Re: [ptxdist] [PATCH v3 1/3] config/setup: make reproducible builds configurable

["Baeuerle, Florian" <Florian.Baeuerle@allegion.com>]

Hi,

we now use this for a while and so far it works well for barebox (there's a
patch required for barebox to get a reproducible binary independent of the
filsystem in use, but that's included since barebox 2019.01.0).

I am not ultimately happy with the patch (ptxdist --version prints a warning),
especially not with

ifndef PTXCONF_SETUP_DISABLE_REPRODUCIBLE

in the rules.

I'll fix the ptxdist --version warning and maybe add a symbol that negates the
above symbol.

I'd be happy about every suggestion that increases the likelihood of acceptance
of the patch. We really need this for reducing bootloader update frequency with
RAUC.


- Florian

Am Freitag, den 21.12.2018, 10:06 +0000 schrieb Baeuerle, Florian:
> Some of ptxdist's packages use SOURCE_DATE_EPOCH to make the build
> results predictable. Make this behaviour more configurable via a newly
> introduced ptxdist setup options.
> 
> By default, this will set SOURCE_DATE_EPOCH to the year and month of the
> used OSELAS Toolchain version. If the used toolchain is not an
> OSELAS-Toolchain, the PTXdist version is used as a fallback.
> 
> Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> ---
>  config/setup/Kconfig                 | 47 ++++++++++++++++++++++++++
>  config/setup/ptxdistrc.default       |  5 +++
>  scripts/lib/ptxd_lib_reproducible.sh | 49 +++++++++++++++++++++++++++-
>  3 files changed, 100 insertions(+), 1 deletion(-)
> 
> diff --git a/config/setup/Kconfig b/config/setup/Kconfig
> index 990da03a1..10adb970d 100644
> --- a/config/setup/Kconfig
> +++ b/config/setup/Kconfig
> @@ -264,6 +264,53 @@ config SETUP_DISABLE_LOCAL_CHECK
>  	  may fail under certain circumstances.
>  	  Disable this check at your own risk.
>  
> +config SETUP_DISABLE_REPRODUCIBLE
> +	bool
> +	prompt "disable reproducible builds"
> +	help
> +	  By default ptxdist will build some packages in a
> +	  reproducible way by injecting fake timestamps, user and
> +	  host name into the build.
> +
> +if !SETUP_DISABLE_REPRODUCIBLE
> +
> +choice
> +	prompt "Fake timestamps source"
> +	default SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN
> +
> +	config SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN
> +		bool
> +		prompt "toolchain version"
> +
> +	config SETUP_REPRODUCIBLE_TIMESTAMP_PTXDIST
> +		bool
> +		prompt "ptxdist version"
> +
> +	config SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM
> +		bool
> +		prompt "custom timestamp"
> +
> +endchoice
> +
> +config SETUP_REPRODUCIBLE_TIMESTAMP
> +	string
> +	default "toolchain" if SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN
> +	default "ptxdist" if SETUP_REPRODUCIBLE_TIMESTAMP_PTXDIST
> +	default "custom" if SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM
> +
> +if SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM
> +
> +config SETUP_REPRODUCIBLE_TIMESTAMP_STRING
> +	string
> +	prompt "Fake timestamp"
> +	default "2018-11-01 UTC"
> +	help
> +	  Supply a custom fake timestamp to be injected to the build.
> +	  The timestamp is passed to 'date --date'.
> +
> +endif
> +endif
> +
>  config SETUP_ENV_WHITELIST
>  	string "environment variable whitelist (space separated)"
>  	help
> diff --git a/config/setup/ptxdistrc.default b/config/setup/ptxdistrc.default
> index 397b78eb7..8aa7dcc9a 100644
> --- a/config/setup/ptxdistrc.default
> +++ b/config/setup/ptxdistrc.default
> @@ -61,6 +61,11 @@ PTXCONF_SETUP_JAVA_SDK="/usr/lib/jvm/default-java"
>  # Developer Options 
>  #
>  # PTXCONF_SETUP_DISABLE_LOCAL_CHECK is not set
> +# PTXCONF_SETUP_DISABLE_REPRODUCIBLE is not set
> +PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN=y
> +# PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_PTXDIST is not set
> +# PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM is not set
> +PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP="toolchain"
>  PTXCONF_SETUP_ENV_WHITELIST=""
>  # PTXCONF_SETUP_COMMON_CACHE is not set
>  # PTXCONF_SETUP_GEN_DEP_TREE is not set
> diff --git a/scripts/lib/ptxd_lib_reproducible.sh
> b/scripts/lib/ptxd_lib_reproducible.sh
> index e2e664ba8..98c528ac2 100644
> --- a/scripts/lib/ptxd_lib_reproducible.sh
> +++ b/scripts/lib/ptxd_lib_reproducible.sh
> @@ -8,8 +8,55 @@
>  # see the README file.
>  #
>  
> +ptxd_timestamp_ptxdist() {
> +    ptxd_reply="${PTXDIST_VERSION_YEAR}-${PTXDIST_VERSION_MONTH}-01 UTC"
> +}
> +
> +ptxd_timestamp_toolchain() {
> +    local oselas_ptxconfig="$(readlink -f "${PTXDIST_TOOLCHAIN}/ptxconfig")"
> +
> +    if [ -e "${oselas_ptxconfig}" ]; then
> +        local oselas_version="$(source "${oselas_ptxconfig}" && echo
> ${PTXCONF_CONFIGFILE_VERSION})"
> +        local orig_IFS="${IFS}"
> +        local IFS="."
> +        set -- ${oselas_version}
> +        IFS="${orig_IFS}"
> +        ptxd_reply="${1}-${2}-01 UTC"
> +    else
> +        echo "${PTXDIST_LOG_PROMPT}warning: cannot deduce timestamp from
> toolchain, falling back to PTXdist version for reproducible timestamp"
> +        ptxd_timestamp_ptxdist
> +    fi
> +}
> +
> +ptxd_timestamp_custom() {
> +    local ts="${PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_STRING}"
> +
> +    if ! date --date "${ts}" > /dev/null 2>&1; then
> +        echo "${PTXDIST_LOG_PROMPT}warning: '${ts}' is not a valid timestamp,
> falling back to toolchain for reproducible timestamp"
> +        ptxd_timestamp_toolchain
> +    else
> +        ptxd_reply="${ts}"
> +    fi
> +}
> +
>  ptxd_lib_reproducible() {
> -    SOURCE_DATE_EPOCH="$(echo $(date --date="${PTXDIST_VERSION_YEAR}-
> ${PTXDIST_VERSION_MONTH}-01 UTC" "+%s"))"
> +    if [ "${PTXCONF_SETUP_DISABLE_REPRODUCIBLE}" = "y" ]; then
> +        ptxd_timestamp_ptxdist
> +    else
> +        case "${PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP}" in
> +            "custom")
> +                ptxd_timestamp_custom
> +                ;;
> +            "ptxdist")
> +                ptxd_timestamp_ptxdist
> +                ;;
> +            *)
> +                ptxd_timestamp_toolchain
> +                ;;
> +        esac
> +    fi
> +
> +    SOURCE_DATE_EPOCH="$(echo $(date --date="${ptxd_reply}" "+%s"))"
>      export SOURCE_DATE_EPOCH
>  
>      PTXDIST_BUILD_TIMESTAMP="$(echo $(date --utc --date @${SOURCE_DATE_EPOCH}
> +%Y-%m-%dT%H:%M+0000))"
> -- 
> 2.19.2
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de