[ptxdist] [PATCH 1/3] host-faketime: new package

[ptxdist] [PATCH 1/3] host-faketime: new package

[Baeuerle, Florian]

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 rules/host-faketime.in   | 10 +++++++++
 rules/host-faketime.make | 47 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 rules/host-faketime.in
 create mode 100644 rules/host-faketime.make

diff --git a/rules/host-faketime.in b/rules/host-faketime.in
new file mode 100644
index 000000000..1a2e7ae19
--- /dev/null
+++ b/rules/host-faketime.in
@@ -0,0 +1,10 @@
+## SECTION=hosttools_noprompt
+## SECTION=hosttools_platform
+
+config HOST_FAKETIME
+	tristate
+	default ALLYES
+	help
+	  libfaketime intercepts various system calls that programs use to
+	  retrieve the current date and time. It then reports modified (faked)
+	  dates and times to these programs.
diff --git a/rules/host-faketime.make b/rules/host-faketime.make
new file mode 100644
index 000000000..462854eb9
--- /dev/null
+++ b/rules/host-faketime.make
@@ -0,0 +1,47 @@
+# -*-makefile-*-
+#
+# Copyright (C) 2018 by Florian Bäuerle <florian.baeuerle@allegion.com>
+#
+# See CREDITS for details about who has contributed to this project.
+#
+# For further information about the PTXdist project and license conditions
+# see the README file.
+#
+
+#
+# We provide this package
+#
+HOST_PACKAGES-$(PTXCONF_HOST_FAKETIME) += host-faketime
+
+#
+# Paths and names
+#
+HOST_FAKETIME_VERSION	= $(LIBFAKETIME_VERSION)
+HOST_FAKETIME_MD5	= $(LIBFAKETIME_MD5)
+HOST_FAKETIME		= $(LIBFAKETIME)
+HOST_FAKETIME_SUFFIX	= $(LIBFAKETIME_SUFFIX)
+HOST_FAKETIME_URL	= $(LIBFAKETIME_URL)
+HOST_FAKETIME_SOURCE	= $(LIBFAKETIME_SOURCE)
+HOST_FAKETIME_LICENSE	= $(LIBFAKETIME_LICENSE)
+HOST_FAKETIME_DIR	= $(HOST_BUILDDIR)/$(HOST_FAKETIME)
+
+# ----------------------------------------------------------------------------
+# Prepare
+# ----------------------------------------------------------------------------
+
+HOST_FAKETIME_CONF_TOOL	= NO
+
+# ----------------------------------------------------------------------------
+# Compile
+# ----------------------------------------------------------------------------
+
+HOST_FAKETIME_MAKE_ENV	= $(HOST_ENV)
+HOST_FAKETIME_MAKE_OPT	= PREFIX=$(PTXCONF_SYSROOT_HOST) DESTDIR=
+
+# ----------------------------------------------------------------------------
+# Install
+# ----------------------------------------------------------------------------
+
+HOST_FAKETIME_INSTALL_OPT	= PREFIX=$(PTXCONF_SYSROOT_HOST) DESTDIR= install
+
+# vim: syntax=make
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Inject fake timestamps into the build via faketime to produce the exact
same binaries.

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 platforms/barebox.in | 10 ++++++++++
 rules/barebox.make   | 16 +++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/platforms/barebox.in b/platforms/barebox.in
index 645366e93..5bc2f08b6 100644
--- a/platforms/barebox.in
+++ b/platforms/barebox.in
@@ -13,6 +13,7 @@ menuconfig BAREBOX
 	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
 	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
 	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
+	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
 	prompt "barebox                       "
 	bool
 	help
@@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
 	  Directories and files to add to the builtin barebox environment.
 	  Relative paths must be relative to the platformconfigdir.
 
+config BAREBOX_REPRODUCIBLE
+	prompt "build reproducibly"
+	bool
+	default n
+	help
+	  With this, fake timestamps are injected to the barebox build, so the
+	  resulting barebox images are identical when the sources haven't been
+	  changed.
+
 config BAREBOX_BAREBOXENV
 	prompt "install 'bareboxenv'"
 	bool
diff --git a/rules/barebox.make b/rules/barebox.make
index a6ec3c394..ad160b3ba 100644
--- a/rules/barebox.make
+++ b/rules/barebox.make
@@ -45,6 +45,20 @@ BAREBOX_ENV := \
 	KCONFIG_NOTIMESTAMP=1 \
 	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
 
+ifdef PTXCONF_BAREBOX_REPRODUCIBLE
+BAREBOX_FAKE_TIMESTAMP := \
+	$(shell echo $(BAREBOX_VERSION) | \
+		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
+BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
+BAREBOX_ENV := $(BAREBOX_ENV) \
+	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
+	KBUILD_BUILD_VERSION="0" \
+	KBUILD_BUILD_USER="ptxdist" \
+	KBUILD_BUILD_HOST="ptxdist"
+else
+BAREBOX_FAKETIME :=
+endif
+
 BAREBOX_MAKEVARS := \
 	V=$(PTXDIST_VERBOSE) \
 	HOSTCC=$(HOSTCC) \
@@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
 endif
 
 	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
-		$(MAKE) $(BAREBOX_MAKEVARS)
+		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
 	@$(call touch)
 
 # ----------------------------------------------------------------------------
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

I tested this for phycore-am335x and phycard-imx27 bootloaders. I get the same
binaries from different machines, but I think anyone using that feature should
first test it before putting any trust into it.

This is a quick implementation that I primarily need for my RAUC bundles (for
not overwriting unchanged bootloaders).

I think this could me implemented as a ptxdist- or platform option that would
theoretically allow building every package reproducibly.

Some packages need special treatment of course, but that could be done with:

ifdef PTXDIST_REPRODUCIBLE
PKG_ENV := $(PKG_ENV) SOME_OTHER_VAR=1
endif

or a new variable:

PKG_REPRODUCIBLE_ENV := ...

Any thoughts or ideas on this?


-Florian

Am Donnerstag, den 11.10.2018, 14:19 +0000 schrieb Baeuerle, Florian:
> Inject fake timestamps into the build via faketime to produce the exact
> same binaries.
> 
> Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> ---
>  platforms/barebox.in | 10 ++++++++++
>  rules/barebox.make   | 16 +++++++++++++++-
>  2 files changed, 25 insertions(+), 1 deletion(-)
> 
> diff --git a/platforms/barebox.in b/platforms/barebox.in
> index 645366e93..5bc2f08b6 100644
> --- a/platforms/barebox.in
> +++ b/platforms/barebox.in
> @@ -13,6 +13,7 @@ menuconfig BAREBOX
>  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
>  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
>  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
>  	prompt "barebox                       "
>  	bool
>  	help
> @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
>  	  Directories and files to add to the builtin barebox environment.
>  	  Relative paths must be relative to the platformconfigdir.
>  
> +config BAREBOX_REPRODUCIBLE
> +	prompt "build reproducibly"
> +	bool
> +	default n
> +	help
> +	  With this, fake timestamps are injected to the barebox build, so the
> +	  resulting barebox images are identical when the sources haven't been
> +	  changed.
> +
>  config BAREBOX_BAREBOXENV
>  	prompt "install 'bareboxenv'"
>  	bool
> diff --git a/rules/barebox.make b/rules/barebox.make
> index a6ec3c394..ad160b3ba 100644
> --- a/rules/barebox.make
> +++ b/rules/barebox.make
> @@ -45,6 +45,20 @@ BAREBOX_ENV := \
>  	KCONFIG_NOTIMESTAMP=1 \
>  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
>  
> +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> +BAREBOX_FAKE_TIMESTAMP := \
> +	$(shell echo $(BAREBOX_VERSION) | \
> +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> +BAREBOX_ENV := $(BAREBOX_ENV) \
> +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> +	KBUILD_BUILD_VERSION="0" \
> +	KBUILD_BUILD_USER="ptxdist" \
> +	KBUILD_BUILD_HOST="ptxdist"
> +else
> +BAREBOX_FAKETIME :=
> +endif
> +
>  BAREBOX_MAKEVARS := \
>  	V=$(PTXDIST_VERBOSE) \
>  	HOSTCC=$(HOSTCC) \
> @@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
>  endif
>  
>  	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
> -		$(MAKE) $(BAREBOX_MAKEVARS)
> +		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
>  	@$(call touch)
>  
>  # -------------------------------------------------------------------------
> ---
> -- 
> 2.19.0
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> Inject fake timestamps into the build via faketime to produce the exact
> same binaries.
> 
> Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> ---
>  platforms/barebox.in | 10 ++++++++++
>  rules/barebox.make   | 16 +++++++++++++++-
>  2 files changed, 25 insertions(+), 1 deletion(-)
> 
> diff --git a/platforms/barebox.in b/platforms/barebox.in
> index 645366e93..5bc2f08b6 100644
> --- a/platforms/barebox.in
> +++ b/platforms/barebox.in
> @@ -13,6 +13,7 @@ menuconfig BAREBOX
>  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
>  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
>  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
>  	prompt "barebox                       "
>  	bool
>  	help
> @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
>  	  Directories and files to add to the builtin barebox environment.
>  	  Relative paths must be relative to the platformconfigdir.
>  
> +config BAREBOX_REPRODUCIBLE
> +	prompt "build reproducibly"
> +	bool
> +	default n
> +	help
> +	  With this, fake timestamps are injected to the barebox build, so the
> +	  resulting barebox images are identical when the sources haven't been
> +	  changed.
> +
>  config BAREBOX_BAREBOXENV
>  	prompt "install 'bareboxenv'"
>  	bool
> diff --git a/rules/barebox.make b/rules/barebox.make
> index a6ec3c394..ad160b3ba 100644
> --- a/rules/barebox.make
> +++ b/rules/barebox.make
> @@ -45,6 +45,20 @@ BAREBOX_ENV := \
>  	KCONFIG_NOTIMESTAMP=1 \
>  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
>  
> +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> +BAREBOX_FAKE_TIMESTAMP := \
> +	$(shell echo $(BAREBOX_VERSION) | \
> +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> +BAREBOX_ENV := $(BAREBOX_ENV) \
> +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> +	KBUILD_BUILD_VERSION="0" \
> +	KBUILD_BUILD_USER="ptxdist" \
> +	KBUILD_BUILD_HOST="ptxdist"

What is not covered by the KBUILD_BUILD_* variables? faketime is a big
hammer that can cause other problems. I'd like to understand what happens
here first.

Michael

> +else
> +BAREBOX_FAKETIME :=
> +endif
> +
>  BAREBOX_MAKEVARS := \
>  	V=$(PTXDIST_VERBOSE) \
>  	HOSTCC=$(HOSTCC) \
> @@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
>  endif
>  
>  	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
> -		$(MAKE) $(BAREBOX_MAKEVARS)
> +		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
>  	@$(call touch)
>  
>  # ----------------------------------------------------------------------------
> -- 
> 2.19.0
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > Inject fake timestamps into the build via faketime to produce the exact
> > same binaries.
> > 
> > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > ---
> >  platforms/barebox.in | 10 ++++++++++
> >  rules/barebox.make   | 16 +++++++++++++++-
> >  2 files changed, 25 insertions(+), 1 deletion(-)
> > 
> > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > index 645366e93..5bc2f08b6 100644
> > --- a/platforms/barebox.in
> > +++ b/platforms/barebox.in
> > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> >  	prompt "barebox                       "
> >  	bool
> >  	help
> > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> >  	  Directories and files to add to the builtin barebox environment.
> >  	  Relative paths must be relative to the platformconfigdir.
> >  
> > +config BAREBOX_REPRODUCIBLE
> > +	prompt "build reproducibly"
> > +	bool
> > +	default n
> > +	help
> > +	  With this, fake timestamps are injected to the barebox build, so the
> > +	  resulting barebox images are identical when the sources haven't been
> > +	  changed.
> > +
> >  config BAREBOX_BAREBOXENV
> >  	prompt "install 'bareboxenv'"
> >  	bool
> > diff --git a/rules/barebox.make b/rules/barebox.make
> > index a6ec3c394..ad160b3ba 100644
> > --- a/rules/barebox.make
> > +++ b/rules/barebox.make
> > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> >  	KCONFIG_NOTIMESTAMP=1 \
> >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> >  
> > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > +BAREBOX_FAKE_TIMESTAMP := \
> > +	$(shell echo $(BAREBOX_VERSION) | \
> > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > +	KBUILD_BUILD_VERSION="0" \
> > +	KBUILD_BUILD_USER="ptxdist" \
> > +	KBUILD_BUILD_HOST="ptxdist"
> 
> What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> hammer that can cause other problems. I'd like to understand what happens
> here first.

According to my observations, only the compression. A few compressors are
capable of not including timestamps (e.g. gzip: --no-name), but others are not
(lzop).

To fix that, there are a few options I do not like:

1. fix the in the barebox build system and implement ugly hacks so this works
for lzop (overwriting an lzo header field) and other compressors that do not
support this out of the box.

2. fix this in ptxdist, probably by doing the same ugly stuff before copying
images to IMAGEDIR

3. fix the problems in lzop and other compressors that do not support faking or
skipping those header fields.

The thirds option is still the best, but it will take ages to reach
distributions.


- Florian

> 
> Michael
> 
> > +else
> > +BAREBOX_FAKETIME :=
> > +endif
> > +
> >  BAREBOX_MAKEVARS := \
> >  	V=$(PTXDIST_VERBOSE) \
> >  	HOSTCC=$(HOSTCC) \
> > @@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
> >  endif
> >  
> >  	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
> > -		$(MAKE) $(BAREBOX_MAKEVARS)
> > +		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
> >  	@$(call touch)
> >  
> >  # -----------------------------------------------------------------------
> > -----
> > -- 
> > 2.19.0
> > 
> > _______________________________________________
> > ptxdist mailing list
> > ptxdist@pengutronix.de
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Fri, Oct 12, 2018 at 12:21:14PM +0000, Baeuerle, Florian wrote:
> Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > Inject fake timestamps into the build via faketime to produce the exact
> > > same binaries.
> > > 
> > > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > > ---
> > >  platforms/barebox.in | 10 ++++++++++
> > >  rules/barebox.make   | 16 +++++++++++++++-
> > >  2 files changed, 25 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > > index 645366e93..5bc2f08b6 100644
> > > --- a/platforms/barebox.in
> > > +++ b/platforms/barebox.in
> > > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> > >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> > >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> > >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> > >  	prompt "barebox                       "
> > >  	bool
> > >  	help
> > > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> > >  	  Directories and files to add to the builtin barebox environment.
> > >  	  Relative paths must be relative to the platformconfigdir.
> > >  
> > > +config BAREBOX_REPRODUCIBLE
> > > +	prompt "build reproducibly"
> > > +	bool
> > > +	default n
> > > +	help
> > > +	  With this, fake timestamps are injected to the barebox build, so the
> > > +	  resulting barebox images are identical when the sources haven't been
> > > +	  changed.
> > > +
> > >  config BAREBOX_BAREBOXENV
> > >  	prompt "install 'bareboxenv'"
> > >  	bool
> > > diff --git a/rules/barebox.make b/rules/barebox.make
> > > index a6ec3c394..ad160b3ba 100644
> > > --- a/rules/barebox.make
> > > +++ b/rules/barebox.make
> > > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> > >  	KCONFIG_NOTIMESTAMP=1 \
> > >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> > >  
> > > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > > +BAREBOX_FAKE_TIMESTAMP := \
> > > +	$(shell echo $(BAREBOX_VERSION) | \
> > > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> > > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > > +	KBUILD_BUILD_VERSION="0" \
> > > +	KBUILD_BUILD_USER="ptxdist" \
> > > +	KBUILD_BUILD_HOST="ptxdist"
> > 
> > What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> > hammer that can cause other problems. I'd like to understand what happens
> > here first.
> 
> According to my observations, only the compression. A few compressors are
> capable of not including timestamps (e.g. gzip: --no-name), but others are not
> (lzop).
> 
> To fix that, there are a few options I do not like:
> 
> 1. fix the in the barebox build system and implement ugly hacks so this works
> for lzop (overwriting an lzo header field) and other compressors that do not
> support this out of the box.
> 
> 2. fix this in ptxdist, probably by doing the same ugly stuff before copying
> images to IMAGEDIR
> 
> 3. fix the problems in lzop and other compressors that do not support faking or
> skipping those header fields.
> 
> The thirds option is still the best, but it will take ages to reach
> distributions.

I've looked at this a bit. For gzip, barebox already uses '-n', so that
should work correctly. Most other compressors don't add a timestamp, at
least in my quick tests. What remains is lzop.

We already have a BAREBOX_NEEDS_HOST_LZOP option, so the only thing needed
is a patch. Using SOURCE_DATE_EPOCH for mtime_low/mtime_high if available
(instead of the file time) should be trivial.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 0/2] Barebox reproducible build + ptxdist setup configuration

[Baeuerle, Florian]

I did not manage to finish the patches the way I wanted to, nevertheless
I dedcided to send them in this state, since I will be on vacation the
upcoming three weeks.

The solution with patching lzop is much nicer than using faketime.

I'll rework the patch series next month, or someone else may pick it up
and polish it if there's an urgent need.


 - Florian

 config/setup/Kconfig                               | 47 +++++++++++++++++++++
 config/setup/ptxdistrc.default                     |  5 +++
 .../0001-allow-overriding-modification-time.patch  | 39 +++++++++++++++++
 patches/lzop-1.03/series                           |  3 +-
 platforms/barebox.in                               |  2 +-
 rules/barebox.make                                 |  9 ++++
 scripts/lib/ptxd_lib_reproducible.sh               | 49 +++++++++++++++++++++-
 7 files changed, 150 insertions(+), 4 deletions(-)

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 1/4] config/setup: make reproducible builds configurable

[Baeuerle, Florian]

Some of ptxdist's packages use SOURCE_DATE_EPOCH to make the build
results predictable. Make this behaviour more configurable via a newly
introduced ptxdist setup options.

By default, this will set SOURCE_DATE_EPOCH to the year and month of the
used OSELAS Toolchain version. If the used toolchain is not an
OSELAS-Toolchain, the PTXdist version is used as a fallback.

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 config/setup/Kconfig                 | 47 ++++++++++++++++++++++++++
 config/setup/ptxdistrc.default       |  5 +++
 scripts/lib/ptxd_lib_reproducible.sh | 49 +++++++++++++++++++++++++++-
 3 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/config/setup/Kconfig b/config/setup/Kconfig
index 990da03a1..10adb970d 100644
--- a/config/setup/Kconfig
+++ b/config/setup/Kconfig
@@ -264,6 +264,53 @@ config SETUP_DISABLE_LOCAL_CHECK
 	  may fail under certain circumstances.
 	  Disable this check at your own risk.
 
+config SETUP_DISABLE_REPRODUCIBLE
+	bool
+	prompt "disable reproducible builds"
+	help
+	  By default ptxdist will build some packages in a
+	  reproducible way by injecting fake timestamps, user and
+	  host name into the build.
+
+if !SETUP_DISABLE_REPRODUCIBLE
+
+choice
+	prompt "Fake timestamps source"
+	default SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN
+
+	config SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN
+		bool
+		prompt "toolchain version"
+
+	config SETUP_REPRODUCIBLE_TIMESTAMP_PTXDIST
+		bool
+		prompt "ptxdist version"
+
+	config SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM
+		bool
+		prompt "custom timestamp"
+
+endchoice
+
+config SETUP_REPRODUCIBLE_TIMESTAMP
+	string
+	default "toolchain" if SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN
+	default "ptxdist" if SETUP_REPRODUCIBLE_TIMESTAMP_PTXDIST
+	default "custom" if SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM
+
+if SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM
+
+config SETUP_REPRODUCIBLE_TIMESTAMP_STRING
+	string
+	prompt "Fake timestamp"
+	default "2018-11-01 UTC"
+	help
+	  Supply a custom fake timestamp to be injected to the build.
+	  The timestamp is passed to 'date --date'.
+
+endif
+endif
+
 config SETUP_ENV_WHITELIST
 	string "environment variable whitelist (space separated)"
 	help
diff --git a/config/setup/ptxdistrc.default b/config/setup/ptxdistrc.default
index 397b78eb7..8aa7dcc9a 100644
--- a/config/setup/ptxdistrc.default
+++ b/config/setup/ptxdistrc.default
@@ -61,6 +61,11 @@ PTXCONF_SETUP_JAVA_SDK="/usr/lib/jvm/default-java"
 # Developer Options 
 #
 # PTXCONF_SETUP_DISABLE_LOCAL_CHECK is not set
+# PTXCONF_SETUP_DISABLE_REPRODUCIBLE is not set
+PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_TOOLCHAIN=y
+# PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_PTXDIST is not set
+# PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_CUSTOM is not set
+PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP="toolchain"
 PTXCONF_SETUP_ENV_WHITELIST=""
 # PTXCONF_SETUP_COMMON_CACHE is not set
 # PTXCONF_SETUP_GEN_DEP_TREE is not set
diff --git a/scripts/lib/ptxd_lib_reproducible.sh b/scripts/lib/ptxd_lib_reproducible.sh
index f0c1bcf8f..06e00e00b 100644
--- a/scripts/lib/ptxd_lib_reproducible.sh
+++ b/scripts/lib/ptxd_lib_reproducible.sh
@@ -8,8 +8,55 @@
 # see the README file.
 #
 
+ptxd_timestamp_ptxdist() {
+    ptxd_reply="${PTXDIST_VERSION_YEAR}-${PTXDIST_VERSION_MONTH}-01 UTC"
+}
+
+ptxd_timestamp_toolchain() {
+    local oselas_ptxconfig="$(readlink -f "${PTXDIST_TOOLCHAIN}/ptxconfig")"
+
+    if [ -e "${oselas_ptxconfig}" ]; then
+        local oselas_version="$(source "${oselas_ptxconfig}" && echo ${PTXCONF_CONFIGFILE_VERSION})"
+        local orig_IFS="${IFS}"
+        local IFS="."
+        set -- ${oselas_version}
+        IFS="${orig_IFS}"
+        ptxd_reply="${1}-${2}-01 UTC"
+    else
+        echo "${PTXDIST_LOG_PROMPT}warning: cannot deduce timestamp from toolchain, falling back to PTXdist version for reproducible timestamp"
+        ptxd_timestamp_ptxdist
+    fi
+}
+
+ptxd_timestamp_custom() {
+    local ts="${PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP_STRING}"
+
+    if ! date --date "${ts}" > /dev/null 2>&1; then
+        echo "${PTXDIST_LOG_PROMPT}warning: '${ts}' is not a valid timestamp, falling back to toolchain for reproducible timestamp"
+        ptxd_timestamp_toolchain
+    else
+        ptxd_reply="${ts}"
+    fi
+}
+
 ptxd_lib_reproducible() {
-    SOURCE_DATE_EPOCH="$(echo $(date --date="${PTXDIST_VERSION_YEAR}-${PTXDIST_VERSION_MONTH}-01 UTC" "+%s"))"
+    if [ "${PTXCONF_SETUP_DISABLE_REPRODUCIBLE}" = "y" ]; then
+        ptxd_timestamp_ptxdist
+    else
+        case "${PTXCONF_SETUP_REPRODUCIBLE_TIMESTAMP}" in
+            "custom")
+                ptxd_timestamp_custom
+                ;;
+            "ptxdist")
+                ptxd_timestamp_ptxdist
+                ;;
+            *)
+                ptxd_timestamp_toolchain
+                ;;
+        esac
+    fi
+
+    SOURCE_DATE_EPOCH="$(echo $(date --date="${ptxd_reply}" "+%s"))"
     export SOURCE_DATE_EPOCH
 }
 export -f ptxd_lib_reproducible
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 2/4] barebox: depend on HOST_LZOP

[Baeuerle, Florian]

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 platforms/barebox.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platforms/barebox.in b/platforms/barebox.in
index 645366e93..46ab32a07 100644
--- a/platforms/barebox.in
+++ b/platforms/barebox.in
@@ -12,7 +12,7 @@ menuconfig BAREBOX
 	select HOST_LIBUSB if BAREBOX_NEEDS_HOST_LIBUSB
 	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
 	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
-	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
+	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP || !SETUP_DISABLE_REPRODUCIBLE
 	prompt "barebox                       "
 	bool
 	help
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 3/4] lzop: add patch that allows overriding mtime

[Baeuerle, Florian]

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 ...1-allow-overriding-modification-time.patch | 39 +++++++++++++++++++
 patches/lzop-1.03/series                      |  3 +-
 2 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 patches/lzop-1.03/0001-allow-overriding-modification-time.patch

diff --git a/patches/lzop-1.03/0001-allow-overriding-modification-time.patch b/patches/lzop-1.03/0001-allow-overriding-modification-time.patch
new file mode 100644
index 000000000..0bc35aaef
--- /dev/null
+++ b/patches/lzop-1.03/0001-allow-overriding-modification-time.patch
@@ -0,0 +1,39 @@
+From af0c84c088c4c1f4a91d6404d4fb02e94f2d1779 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Florian=20B=C3=A4uerle?= <florian.baeuerle@allegion.com>
+Date: Thu, 25 Oct 2018 17:26:30 +0200
+Subject: [PATCH] allow overriding modification time
+
+---
+ src/lzop.c | 9 ++++++++-
+ 1 file changed, 8 insertions(+), 1 deletion(-)
+
+diff --git a/src/lzop.c b/src/lzop.c
+index 4cd1bb7..e3dba51 100644
+--- a/src/lzop.c
++++ b/src/lzop.c
+@@ -706,6 +706,7 @@ void init_compress_header(header_t *h, const file_t *fip, const file_t *fop)
+     assert(opt_method > 0);
+     assert(opt_level > 0);
+     assert(fip->st.st_mode == 0 || S_ISREG(fip->st.st_mode));
++    const char *source_date_epoch = getenv("SOURCE_DATE_EPOCH");
+ 
+     memset(h,0,sizeof(header_t));
+ 
+@@ -742,7 +743,13 @@ void init_compress_header(header_t *h, const file_t *fip, const file_t *fop)
+ 
+     h->mode = fix_mode_for_header(fip->st.st_mode);
+ 
+-    if (fip->st.st_mtime)
++    if (source_date_epoch)
++    {
++        time_t mtime = strtoul(source_date_epoch, NULL, 0);
++        h->mtime_low  = (lzo_uint32) (mtime);
++        h->mtime_high = (lzo_uint32) ((mtime >> 16) >> 16);
++    }
++    else if (fip->st.st_mtime)
+     {
+         h->mtime_low = (lzo_uint32) (fip->st.st_mtime);
+         h->mtime_high = (lzo_uint32) (fip->st.st_mtime >> 16 >> 16);
+-- 
+1.8.3.1
+
diff --git a/patches/lzop-1.03/series b/patches/lzop-1.03/series
index ec5e32346..272ffc22f 100644
--- a/patches/lzop-1.03/series
+++ b/patches/lzop-1.03/series
@@ -1,3 +1,2 @@
-# generated by git-ptx-patches
 0001-make-block-size-configureable.patch
-# 81ee93089124310f55464ef8b543428c  - git-ptx-patches magic
+0001-allow-overriding-modification-time.patch
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 4/4] barebox: add support for reproducible build

[Baeuerle, Florian]

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 rules/barebox.make | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/rules/barebox.make b/rules/barebox.make
index a6ec3c394..68f1c6203 100644
--- a/rules/barebox.make
+++ b/rules/barebox.make
@@ -45,6 +45,15 @@ BAREBOX_ENV := \
 	KCONFIG_NOTIMESTAMP=1 \
 	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
 
+ifndef PTXCONF_SETUP_DISABLE_REPRODUCIBLE
+BAREBOX_ENV := $(BAREBOX_ENV) \
+	SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
+	KBUILD_BUILD_TIMESTAMP="$(shell date --utc --date @$(SOURCE_DATE_EPOCH))" \
+	KBUILD_BUILD_VERSION="0" \
+	KBUILD_BUILD_USER="ptxdist" \
+	KBUILD_BUILD_HOST="ptxdist"
+endif
+
 BAREBOX_MAKEVARS := \
 	V=$(PTXDIST_VERBOSE) \
 	HOSTCC=$(HOSTCC) \
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > Inject fake timestamps into the build via faketime to produce the exact
> > same binaries.
> > 
> > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > ---
> >  platforms/barebox.in | 10 ++++++++++
> >  rules/barebox.make   | 16 +++++++++++++++-
> >  2 files changed, 25 insertions(+), 1 deletion(-)
> > 
> > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > index 645366e93..5bc2f08b6 100644
> > --- a/platforms/barebox.in
> > +++ b/platforms/barebox.in
> > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> >  	prompt "barebox                       "
> >  	bool
> >  	help
> > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> >  	  Directories and files to add to the builtin barebox environment.
> >  	  Relative paths must be relative to the platformconfigdir.
> >  
> > +config BAREBOX_REPRODUCIBLE
> > +	prompt "build reproducibly"
> > +	bool
> > +	default n
> > +	help
> > +	  With this, fake timestamps are injected to the barebox build, so the
> > +	  resulting barebox images are identical when the sources haven't been
> > +	  changed.
> > +
> >  config BAREBOX_BAREBOXENV
> >  	prompt "install 'bareboxenv'"
> >  	bool
> > diff --git a/rules/barebox.make b/rules/barebox.make
> > index a6ec3c394..ad160b3ba 100644
> > --- a/rules/barebox.make
> > +++ b/rules/barebox.make
> > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> >  	KCONFIG_NOTIMESTAMP=1 \
> >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> >  
> > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > +BAREBOX_FAKE_TIMESTAMP := \
> > +	$(shell echo $(BAREBOX_VERSION) | \
> > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > +	KBUILD_BUILD_VERSION="0" \
> > +	KBUILD_BUILD_USER="ptxdist" \
> > +	KBUILD_BUILD_HOST="ptxdist"

For the sake of completeness: the _USER and _HOST variables are *not yet*
incorporated into the resulting binaries by barebox (at least with my
configuration).

> 
> What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> hammer that can cause other problems. I'd like to understand what happens
> here first.

It's an opt-in hammer at least. One annoyance that I observed is, that
menuconfig barebox also does build host-faketime and a bunch of other things
that are not a dependency of the host-faketime package.


- Florian

> 
> Michael
> 
> > +else
> > +BAREBOX_FAKETIME :=
> > +endif
> > +
> >  BAREBOX_MAKEVARS := \
> >  	V=$(PTXDIST_VERBOSE) \
> >  	HOSTCC=$(HOSTCC) \
> > @@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
> >  endif
> >  
> >  	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
> > -		$(MAKE) $(BAREBOX_MAKEVARS)
> > +		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
> >  	@$(call touch)
> >  
> >  # -----------------------------------------------------------------------
> > -----
> > -- 
> > 2.19.0
> > 
> > _______________________________________________
> > ptxdist mailing list
> > ptxdist@pengutronix.de
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Fri, Oct 12, 2018 at 12:28:19PM +0000, Baeuerle, Florian wrote:
> Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > Inject fake timestamps into the build via faketime to produce the exact
> > > same binaries.
> > > 
> > > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > > ---
> > >  platforms/barebox.in | 10 ++++++++++
> > >  rules/barebox.make   | 16 +++++++++++++++-
> > >  2 files changed, 25 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > > index 645366e93..5bc2f08b6 100644
> > > --- a/platforms/barebox.in
> > > +++ b/platforms/barebox.in
> > > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> > >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> > >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> > >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> > >  	prompt "barebox                       "
> > >  	bool
> > >  	help
> > > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> > >  	  Directories and files to add to the builtin barebox environment.
> > >  	  Relative paths must be relative to the platformconfigdir.
> > >  
> > > +config BAREBOX_REPRODUCIBLE
> > > +	prompt "build reproducibly"
> > > +	bool
> > > +	default n
> > > +	help
> > > +	  With this, fake timestamps are injected to the barebox build, so the
> > > +	  resulting barebox images are identical when the sources haven't been
> > > +	  changed.
> > > +
> > >  config BAREBOX_BAREBOXENV
> > >  	prompt "install 'bareboxenv'"
> > >  	bool
> > > diff --git a/rules/barebox.make b/rules/barebox.make
> > > index a6ec3c394..ad160b3ba 100644
> > > --- a/rules/barebox.make
> > > +++ b/rules/barebox.make
> > > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> > >  	KCONFIG_NOTIMESTAMP=1 \
> > >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> > >  
> > > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > > +BAREBOX_FAKE_TIMESTAMP := \
> > > +	$(shell echo $(BAREBOX_VERSION) | \
> > > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> > > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > > +	KBUILD_BUILD_VERSION="0" \
> > > +	KBUILD_BUILD_USER="ptxdist" \
> > > +	KBUILD_BUILD_HOST="ptxdist"
> 
> For the sake of completeness: the _USER and _HOST variables are *not yet*
> incorporated into the resulting binaries by barebox (at least with my
> configuration).

Hmm, then something is wrong here. scripts/mkcompile_h has always used
these variables.

> > 
> > What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> > hammer that can cause other problems. I'd like to understand what happens
> > here first.
> 
> It's an opt-in hammer at least. One annoyance that I observed is, that
> menuconfig barebox also does build host-faketime and a bunch of other things
> that are not a dependency of the host-faketime package.

That's the usual early build stuff, right? host-pkg-config and things like
that.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Am Dienstag, den 16.10.2018, 10:59 +0200 schrieb Michael Olbrich:
> On Fri, Oct 12, 2018 at 12:28:19PM +0000, Baeuerle, Florian wrote:
> > Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> > > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > > Inject fake timestamps into the build via faketime to produce the exact
> > > > same binaries.
> > > > 
> > > > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > > > ---
> > > >  platforms/barebox.in | 10 ++++++++++
> > > >  rules/barebox.make   | 16 +++++++++++++++-
> > > >  2 files changed, 25 insertions(+), 1 deletion(-)
> > > > 
> > > > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > > > index 645366e93..5bc2f08b6 100644
> > > > --- a/platforms/barebox.in
> > > > +++ b/platforms/barebox.in
> > > > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> > > >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> > > >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> > > >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > > > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> > > >  	prompt "barebox                       "
> > > >  	bool
> > > >  	help
> > > > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> > > >  	  Directories and files to add to the builtin barebox
> > > > environment.
> > > >  	  Relative paths must be relative to the platformconfigdir.
> > > >  
> > > > +config BAREBOX_REPRODUCIBLE
> > > > +	prompt "build reproducibly"
> > > > +	bool
> > > > +	default n
> > > > +	help
> > > > +	  With this, fake timestamps are injected to the barebox build,
> > > > so the
> > > > +	  resulting barebox images are identical when the sources
> > > > haven't been
> > > > +	  changed.
> > > > +
> > > >  config BAREBOX_BAREBOXENV
> > > >  	prompt "install 'bareboxenv'"
> > > >  	bool
> > > > diff --git a/rules/barebox.make b/rules/barebox.make
> > > > index a6ec3c394..ad160b3ba 100644
> > > > --- a/rules/barebox.make
> > > > +++ b/rules/barebox.make
> > > > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> > > >  	KCONFIG_NOTIMESTAMP=1 \
> > > >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> > > >  
> > > > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > > > +BAREBOX_FAKE_TIMESTAMP := \
> > > > +	$(shell echo $(BAREBOX_VERSION) | \
> > > > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01
> > > > 00:00:00/")
> > > > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > > > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > > > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > > > +	KBUILD_BUILD_VERSION="0" \
> > > > +	KBUILD_BUILD_USER="ptxdist" \
> > > > +	KBUILD_BUILD_HOST="ptxdist"
> > 
> > For the sake of completeness: the _USER and _HOST variables are *not yet*
> > incorporated into the resulting binaries by barebox (at least with my
> > configuration).
> 
> Hmm, then something is wrong here. scripts/mkcompile_h has always used
> these variables.

Yes, it generates #defines, but apparently these are not used anywhere in the
barebox source tree, thus they do not end up in the binary.

> 
> > > What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> > > hammer that can cause other problems. I'd like to understand what happens
> > > here first.
> > 
> > It's an opt-in hammer at least. One annoyance that I observed is, that
> > menuconfig barebox also does build host-faketime and a bunch of other things
> > that are not a dependency of the host-faketime package.
> 
> That's the usual early build stuff, right? host-pkg-config and things like
> that.
> 

I guess so. It's host-chrpath, host-pkg-config, kernel-header, host-attr, host-
autotools-*, host-acl, host-fakeroot, host-m4 to name a few.


- Florian
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Tue, Oct 16, 2018 at 11:06:19AM +0000, Baeuerle, Florian wrote:
> Am Dienstag, den 16.10.2018, 10:59 +0200 schrieb Michael Olbrich:
> > On Fri, Oct 12, 2018 at 12:28:19PM +0000, Baeuerle, Florian wrote:
> > > Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> > > > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > > > Inject fake timestamps into the build via faketime to produce the exact
> > > > > same binaries.
> > > > > 
> > > > > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > > > > ---
> > > > >  platforms/barebox.in | 10 ++++++++++
> > > > >  rules/barebox.make   | 16 +++++++++++++++-
> > > > >  2 files changed, 25 insertions(+), 1 deletion(-)
> > > > > 
> > > > > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > > > > index 645366e93..5bc2f08b6 100644
> > > > > --- a/platforms/barebox.in
> > > > > +++ b/platforms/barebox.in
> > > > > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> > > > >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> > > > >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> > > > >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > > > > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> > > > >  	prompt "barebox                       "
> > > > >  	bool
> > > > >  	help
> > > > > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> > > > >  	  Directories and files to add to the builtin barebox
> > > > > environment.
> > > > >  	  Relative paths must be relative to the platformconfigdir.
> > > > >  
> > > > > +config BAREBOX_REPRODUCIBLE
> > > > > +	prompt "build reproducibly"
> > > > > +	bool
> > > > > +	default n
> > > > > +	help
> > > > > +	  With this, fake timestamps are injected to the barebox build,
> > > > > so the
> > > > > +	  resulting barebox images are identical when the sources
> > > > > haven't been
> > > > > +	  changed.
> > > > > +
> > > > >  config BAREBOX_BAREBOXENV
> > > > >  	prompt "install 'bareboxenv'"
> > > > >  	bool
> > > > > diff --git a/rules/barebox.make b/rules/barebox.make
> > > > > index a6ec3c394..ad160b3ba 100644
> > > > > --- a/rules/barebox.make
> > > > > +++ b/rules/barebox.make
> > > > > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> > > > >  	KCONFIG_NOTIMESTAMP=1 \
> > > > >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> > > > >  
> > > > > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > > > > +BAREBOX_FAKE_TIMESTAMP := \
> > > > > +	$(shell echo $(BAREBOX_VERSION) | \
> > > > > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01
> > > > > 00:00:00/")
> > > > > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > > > > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > > > > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > > > > +	KBUILD_BUILD_VERSION="0" \
> > > > > +	KBUILD_BUILD_USER="ptxdist" \
> > > > > +	KBUILD_BUILD_HOST="ptxdist"
> > > 
> > > For the sake of completeness: the _USER and _HOST variables are *not yet*
> > > incorporated into the resulting binaries by barebox (at least with my
> > > configuration).
> > 
> > Hmm, then something is wrong here. scripts/mkcompile_h has always used
> > these variables.
> 
> Yes, it generates #defines, but apparently these are not used anywhere in the
> barebox source tree, thus they do not end up in the binary.

Ok, not used as in no user/host in the binary at all. That's ok then.

> > > > What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> > > > hammer that can cause other problems. I'd like to understand what happens
> > > > here first.
> > > 
> > > It's an opt-in hammer at least. One annoyance that I observed is, that
> > > menuconfig barebox also does build host-faketime and a bunch of other things
> > > that are not a dependency of the host-faketime package.
> > 
> > That's the usual early build stuff, right? host-pkg-config and things like
> > that.
> > 
> 
> I guess so. It's host-chrpath, host-pkg-config, kernel-header, host-attr, host-
> autotools-*, host-acl, host-fakeroot, host-m4 to name a few.

Hmmm, I applied the patches and did some quick checks, and I'm not seeing
the issue. If you run ptxdist with '-d' you might get a hint where the
dependency is coming from.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Am Dienstag, den 16.10.2018, 14:51 +0200 schrieb Michael Olbrich:
> On Tue, Oct 16, 2018 at 11:06:19AM +0000, Baeuerle, Florian wrote:
> > Am Dienstag, den 16.10.2018, 10:59 +0200 schrieb Michael Olbrich:
> > > On Fri, Oct 12, 2018 at 12:28:19PM +0000, Baeuerle, Florian wrote:
> > > > Am Freitag, den 12.10.2018, 14:01 +0200 schrieb Michael Olbrich:
> > > > > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > > > > Inject fake timestamps into the build via faketime to produce the
> > > > > > exact
> > > > > > same binaries.
> > > > > > 
> > > > > > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > > > > > ---
> > > > > >  platforms/barebox.in | 10 ++++++++++
> > > > > >  rules/barebox.make   | 16 +++++++++++++++-
> > > > > >  2 files changed, 25 insertions(+), 1 deletion(-)
> > > > > > 
> > > > > > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > > > > > index 645366e93..5bc2f08b6 100644
> > > > > > --- a/platforms/barebox.in
> > > > > > +++ b/platforms/barebox.in
> > > > > > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> > > > > >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> > > > > >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> > > > > >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > > > > > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> > > > > >  	prompt "barebox                       "
> > > > > >  	bool
> > > > > >  	help
> > > > > > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> > > > > >  	  Directories and files to add to the builtin barebox
> > > > > > environment.
> > > > > >  	  Relative paths must be relative to the platformconfigdir.
> > > > > >  
> > > > > > +config BAREBOX_REPRODUCIBLE
> > > > > > +	prompt "build reproducibly"
> > > > > > +	bool
> > > > > > +	default n
> > > > > > +	help
> > > > > > +	  With this, fake timestamps are injected to the barebox build,
> > > > > > so the
> > > > > > +	  resulting barebox images are identical when the sources
> > > > > > haven't been
> > > > > > +	  changed.
> > > > > > +
> > > > > >  config BAREBOX_BAREBOXENV
> > > > > >  	prompt "install 'bareboxenv'"
> > > > > >  	bool
> > > > > > diff --git a/rules/barebox.make b/rules/barebox.make
> > > > > > index a6ec3c394..ad160b3ba 100644
> > > > > > --- a/rules/barebox.make
> > > > > > +++ b/rules/barebox.make
> > > > > > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> > > > > >  	KCONFIG_NOTIMESTAMP=1 \
> > > > > >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> > > > > >  
> > > > > > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > > > > > +BAREBOX_FAKE_TIMESTAMP := \
> > > > > > +	$(shell echo $(BAREBOX_VERSION) | \
> > > > > > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01
> > > > > > 00:00:00/")
> > > > > > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > > > > > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > > > > > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > > > > > +	KBUILD_BUILD_VERSION="0" \
> > > > > > +	KBUILD_BUILD_USER="ptxdist" \
> > > > > > +	KBUILD_BUILD_HOST="ptxdist"
> > > > 
> > > > For the sake of completeness: the _USER and _HOST variables are *not
> > > > yet*
> > > > incorporated into the resulting binaries by barebox (at least with my
> > > > configuration).
> > > 
> > > Hmm, then something is wrong here. scripts/mkcompile_h has always used
> > > these variables.
> > 
> > Yes, it generates #defines, but apparently these are not used anywhere in
> > the
> > barebox source tree, thus they do not end up in the binary.
> 
> Ok, not used as in no user/host in the binary at all. That's ok then.
> 
> > > > > What is not covered by the KBUILD_BUILD_* variables? faketime is a big
> > > > > hammer that can cause other problems. I'd like to understand what
> > > > > happens
> > > > > here first.
> > > > 
> > > > It's an opt-in hammer at least. One annoyance that I observed is, that
> > > > menuconfig barebox also does build host-faketime and a bunch of other
> > > > things
> > > > that are not a dependency of the host-faketime package.
> > > 
> > > That's the usual early build stuff, right? host-pkg-config and things like
> > > that.
> > > 
> > 
> > I guess so. It's host-chrpath, host-pkg-config, kernel-header, host-attr,
> > host-
> > autotools-*, host-acl, host-fakeroot, host-m4 to name a few.
> 
> Hmmm, I applied the patches and did some quick checks, and I'm not seeing
> the issue. If you run ptxdist with '-d' you might get a hint where the
> dependency is coming from.

It's not an issue, the same happens without the patches. I just never noticed
it. The dependencies come from base.install.


- Florian
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Tue, Oct 16, 2018 at 02:11:52PM +0000, Baeuerle, Florian wrote:
> Am Dienstag, den 16.10.2018, 14:51 +0200 schrieb Michael Olbrich:
> > On Tue, Oct 16, 2018 at 11:06:19AM +0000, Baeuerle, Florian wrote:
> > > Am Dienstag, den 16.10.2018, 10:59 +0200 schrieb Michael Olbrich:
> > > > On Fri, Oct 12, 2018 at 12:28:19PM +0000, Baeuerle, Florian wrote:
> > > > > It's an opt-in hammer at least. One annoyance that I observed is, that
> > > > > menuconfig barebox also does build host-faketime and a bunch of other
> > > > > things
> > > > > that are not a dependency of the host-faketime package.
> > > > 
> > > > That's the usual early build stuff, right? host-pkg-config and things like
> > > > that.
> > > > 
> > > 
> > > I guess so. It's host-chrpath, host-pkg-config, kernel-header, host-attr,
> > > host-
> > > autotools-*, host-acl, host-fakeroot, host-m4 to name a few.
> > 
> > Hmmm, I applied the patches and did some quick checks, and I'm not seeing
> > the issue. If you run ptxdist with '-d' you might get a hint where the
> > dependency is coming from.
> 
> It's not an issue, the same happens without the patches. I just never noticed
> it. The dependencies come from base.install.

So the question is, why does barebox_menuconfig depend on base.install.
Probably via barebox.extract but that shouldn't happen either.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> Inject fake timestamps into the build via faketime to produce the exact
> same binaries.

I'd like to see this patch split in two.
One to handle the KBUILD_BUILD_ stuff. That should be done like the kernel
with the symlink magic, but otherwise unconditionally (for now).
Use SOURCE_DATE_EPOCH for the timestamp. I want to change the kernel to use
this too.

Another patch for the rest. I'm not quite sure how I want to handle this.
My current plan is to have a setup option: Developers can use it to disable
reproducible builds to see stuff like the build timestamps etc.
CI and release builds have it enabled by default. I currently don't have
any reason do disable reproducibility for releases.

What do you think?

Michael

> Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> ---
>  platforms/barebox.in | 10 ++++++++++
>  rules/barebox.make   | 16 +++++++++++++++-
>  2 files changed, 25 insertions(+), 1 deletion(-)
> 
> diff --git a/platforms/barebox.in b/platforms/barebox.in
> index 645366e93..5bc2f08b6 100644
> --- a/platforms/barebox.in
> +++ b/platforms/barebox.in
> @@ -13,6 +13,7 @@ menuconfig BAREBOX
>  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
>  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
>  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
>  	prompt "barebox                       "
>  	bool
>  	help
> @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
>  	  Directories and files to add to the builtin barebox environment.
>  	  Relative paths must be relative to the platformconfigdir.
>  
> +config BAREBOX_REPRODUCIBLE
> +	prompt "build reproducibly"
> +	bool
> +	default n
> +	help
> +	  With this, fake timestamps are injected to the barebox build, so the
> +	  resulting barebox images are identical when the sources haven't been
> +	  changed.
> +
>  config BAREBOX_BAREBOXENV
>  	prompt "install 'bareboxenv'"
>  	bool
> diff --git a/rules/barebox.make b/rules/barebox.make
> index a6ec3c394..ad160b3ba 100644
> --- a/rules/barebox.make
> +++ b/rules/barebox.make
> @@ -45,6 +45,20 @@ BAREBOX_ENV := \
>  	KCONFIG_NOTIMESTAMP=1 \
>  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
>  
> +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> +BAREBOX_FAKE_TIMESTAMP := \
> +	$(shell echo $(BAREBOX_VERSION) | \
> +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> +BAREBOX_ENV := $(BAREBOX_ENV) \
> +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> +	KBUILD_BUILD_VERSION="0" \
> +	KBUILD_BUILD_USER="ptxdist" \
> +	KBUILD_BUILD_HOST="ptxdist"
> +else
> +BAREBOX_FAKETIME :=
> +endif
> +
>  BAREBOX_MAKEVARS := \
>  	V=$(PTXDIST_VERBOSE) \
>  	HOSTCC=$(HOSTCC) \
> @@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
>  endif
>  
>  	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
> -		$(MAKE) $(BAREBOX_MAKEVARS)
> +		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
>  	@$(call touch)
>  
>  # ----------------------------------------------------------------------------
> -- 
> 2.19.0
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Hi Michael,

Am Dienstag, den 16.10.2018, 12:09 +0200 schrieb Michael Olbrich:
> On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > Inject fake timestamps into the build via faketime to produce the exact
> > same binaries.
> 
> I'd like to see this patch split in two.
> One to handle the KBUILD_BUILD_ stuff. That should be done like the kernel
> with the symlink magic, but otherwise unconditionally (for now).
> Use SOURCE_DATE_EPOCH for the timestamp. I want to change the kernel to use
> this too.

SOURCE_DATE_EPOCH uses the ptxdist release number for the timestamp, thus the
timestamp changes with every ptxdist release. I'd prefer to *really* get the
same binary when the source code was not changed at all. A ptxdist version bump
has a high potential of not touching anything barebox related.

I also do not like mangling the barebox version into a timestamp, but I couldn't
come up with any better idea.

> 
> Another patch for the rest. I'm not quite sure how I want to handle this.
> My current plan is to have a setup option: Developers can use it to disable
> reproducible builds to see stuff like the build timestamps etc.
> CI and release builds have it enabled by default. I currently don't have
> any reason do disable reproducibility for releases.
> 
> What do you think?

My main use case for reproducible builds is RAUC. Since bootloader updates are
especially hairy on some platforms, I'm happy about having the ability to spare
a few of them (while retaining an update mechanism that ensures a persistent
state of the system *and* without using prebuilt bootloaders).

However, I do not mind if ptxdist by default builds more packages reproducibly.



-Florian

> 
> Michael
> 
> > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > ---
> >  platforms/barebox.in | 10 ++++++++++
> >  rules/barebox.make   | 16 +++++++++++++++-
> >  2 files changed, 25 insertions(+), 1 deletion(-)
> > 
> > diff --git a/platforms/barebox.in b/platforms/barebox.in
> > index 645366e93..5bc2f08b6 100644
> > --- a/platforms/barebox.in
> > +++ b/platforms/barebox.in
> > @@ -13,6 +13,7 @@ menuconfig BAREBOX
> >  	select HOST_OPENSSL if BAREBOX_NEEDS_HOST_OPENSSL
> >  	select HOST_IMX_CST if BAREBOX_NEEDS_HOST_IMX_CST
> >  	select HOST_LZOP if BAREBOX_NEEDS_HOST_LZOP
> > +	select HOST_FAKETIME if BAREBOX_REPRODUCIBLE
> >  	prompt "barebox                       "
> >  	bool
> >  	help
> > @@ -50,6 +51,15 @@ config BAREBOX_EXTRA_ENV_PATH
> >  	  Directories and files to add to the builtin barebox environment.
> >  	  Relative paths must be relative to the platformconfigdir.
> >  
> > +config BAREBOX_REPRODUCIBLE
> > +	prompt "build reproducibly"
> > +	bool
> > +	default n
> > +	help
> > +	  With this, fake timestamps are injected to the barebox build, so the
> > +	  resulting barebox images are identical when the sources haven't been
> > +	  changed.
> > +
> >  config BAREBOX_BAREBOXENV
> >  	prompt "install 'bareboxenv'"
> >  	bool
> > diff --git a/rules/barebox.make b/rules/barebox.make
> > index a6ec3c394..ad160b3ba 100644
> > --- a/rules/barebox.make
> > +++ b/rules/barebox.make
> > @@ -45,6 +45,20 @@ BAREBOX_ENV := \
> >  	KCONFIG_NOTIMESTAMP=1 \
> >  	pkg_wrapper_blacklist="$(BAREBOX_WRAPPER_BLACKLIST)"
> >  
> > +ifdef PTXCONF_BAREBOX_REPRODUCIBLE
> > +BAREBOX_FAKE_TIMESTAMP := \
> > +	$(shell echo $(BAREBOX_VERSION) | \
> > +		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
> > +BAREBOX_FAKETIME := faketime -f "$(BAREBOX_FAKE_TIMESTAMP)"
> > +BAREBOX_ENV := $(BAREBOX_ENV) \
> > +	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_FAKE_TIMESTAMP)" \
> > +	KBUILD_BUILD_VERSION="0" \
> > +	KBUILD_BUILD_USER="ptxdist" \
> > +	KBUILD_BUILD_HOST="ptxdist"
> > +else
> > +BAREBOX_FAKETIME :=
> > +endif
> > +
> >  BAREBOX_MAKEVARS := \
> >  	V=$(PTXDIST_VERBOSE) \
> >  	HOSTCC=$(HOSTCC) \
> > @@ -113,7 +127,7 @@ ifdef PTXCONF_BAREBOX_EXTRA_ENV
> >  endif
> >  
> >  	@+cd $(BAREBOX_DIR) && $(BAREBOX_PATH) $(BAREBOX_ENV) \
> > -		$(MAKE) $(BAREBOX_MAKEVARS)
> > +		$(BAREBOX_FAKETIME) $(MAKE) $(BAREBOX_MAKEVARS)
> >  	@$(call touch)
> >  
> >  # -----------------------------------------------------------------------
> > -----
> > -- 
> > 2.19.0
> > 
> > _______________________________________________
> > ptxdist mailing list
> > ptxdist@pengutronix.de
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Tue, Oct 16, 2018 at 10:57:20AM +0000, Baeuerle, Florian wrote:
> Hi Michael,
> 
> Am Dienstag, den 16.10.2018, 12:09 +0200 schrieb Michael Olbrich:
> > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > Inject fake timestamps into the build via faketime to produce the exact
> > > same binaries.
> > 
> > I'd like to see this patch split in two.
> > One to handle the KBUILD_BUILD_ stuff. That should be done like the kernel
> > with the symlink magic, but otherwise unconditionally (for now).
> > Use SOURCE_DATE_EPOCH for the timestamp. I want to change the kernel to use
> > this too.
> 
> SOURCE_DATE_EPOCH uses the ptxdist release number for the timestamp, thus the
> timestamp changes with every ptxdist release. I'd prefer to *really* get the
> same binary when the source code was not changed at all. A ptxdist version bump
> has a high potential of not touching anything barebox related.
> 
> I also do not like mangling the barebox version into a timestamp, but I couldn't
> come up with any better idea.

You can easily overwrite that in your BSP and I'm wouldn't mind changing
that in general. The PTXdist release was just a convenient 'date' to use.
I'd like to have something that is global to the BSP. Do you have any
sugestions?

> > Another patch for the rest. I'm not quite sure how I want to handle this.
> > My current plan is to have a setup option: Developers can use it to disable
> > reproducible builds to see stuff like the build timestamps etc.
> > CI and release builds have it enabled by default. I currently don't have
> > any reason do disable reproducibility for releases.
> > 
> > What do you think?
> 
> My main use case for reproducible builds is RAUC. Since bootloader updates are
> especially hairy on some platforms, I'm happy about having the ability to spare
> a few of them (while retaining an update mechanism that ensures a persistent
> state of the system *and* without using prebuilt bootloaders).
> 
> However, I do not mind if ptxdist by default builds more packages reproducibly.

Updates are certainly one use-case. For me it's also useful to develop
PTXdist. I can compare ipkgs from before and after some refactoring more
easily.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Baeuerle, Florian]

Am Dienstag, den 16.10.2018, 14:38 +0200 schrieb Michael Olbrich:
> On Tue, Oct 16, 2018 at 10:57:20AM +0000, Baeuerle, Florian wrote:
> > Hi Michael,
> > 
> > Am Dienstag, den 16.10.2018, 12:09 +0200 schrieb Michael Olbrich:
> > > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > > Inject fake timestamps into the build via faketime to produce the exact
> > > > same binaries.
> > > 
> > > I'd like to see this patch split in two.
> > > One to handle the KBUILD_BUILD_ stuff. That should be done like the kernel
> > > with the symlink magic, but otherwise unconditionally (for now).
> > > Use SOURCE_DATE_EPOCH for the timestamp. I want to change the kernel to
> > > use
> > > this too.
> > 
> > SOURCE_DATE_EPOCH uses the ptxdist release number for the timestamp, thus
> > the
> > timestamp changes with every ptxdist release. I'd prefer to *really* get the
> > same binary when the source code was not changed at all. A ptxdist version
> > bump
> > has a high potential of not touching anything barebox related.
> > 
> > I also do not like mangling the barebox version into a timestamp, but I
> > couldn't
> > come up with any better idea.
> 
> You can easily overwrite that in your BSP and I'm wouldn't mind changing
> that in general. The PTXdist release was just a convenient 'date' to use.
> I'd like to have something that is global to the BSP. Do you have any
> sugestions?

Also I'd prefer to use something BSP-global, but I couldn't find anything
suitable:

PTXCONF_PROJECT_VERSION - not guaranteed to be a date
PTXCONF_PLATFORM_VERSION - same here

Maybe ptxdist could offer using one of those as a fake timestamp, if the
appropriate format is used.


- Florian
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Tue, Oct 16, 2018 at 02:27:05PM +0000, Baeuerle, Florian wrote:
> Am Dienstag, den 16.10.2018, 14:38 +0200 schrieb Michael Olbrich:
> > On Tue, Oct 16, 2018 at 10:57:20AM +0000, Baeuerle, Florian wrote:
> > > Am Dienstag, den 16.10.2018, 12:09 +0200 schrieb Michael Olbrich:
> > > > On Thu, Oct 11, 2018 at 02:19:17PM +0000, Baeuerle, Florian wrote:
> > > > > Inject fake timestamps into the build via faketime to produce the exact
> > > > > same binaries.
> > > > 
> > > > I'd like to see this patch split in two.
> > > > One to handle the KBUILD_BUILD_ stuff. That should be done like the kernel
> > > > with the symlink magic, but otherwise unconditionally (for now).
> > > > Use SOURCE_DATE_EPOCH for the timestamp. I want to change the kernel to
> > > > use
> > > > this too.
> > > 
> > > SOURCE_DATE_EPOCH uses the ptxdist release number for the timestamp, thus
> > > the
> > > timestamp changes with every ptxdist release. I'd prefer to *really* get the
> > > same binary when the source code was not changed at all. A ptxdist version
> > > bump
> > > has a high potential of not touching anything barebox related.
> > > 
> > > I also do not like mangling the barebox version into a timestamp, but I
> > > couldn't
> > > come up with any better idea.
> > 
> > You can easily overwrite that in your BSP and I'm wouldn't mind changing
> > that in general. The PTXdist release was just a convenient 'date' to use.
> > I'd like to have something that is global to the BSP. Do you have any
> > sugestions?
> 
> Also I'd prefer to use something BSP-global, but I couldn't find anything
> suitable:
> 
> PTXCONF_PROJECT_VERSION - not guaranteed to be a date
> PTXCONF_PLATFORM_VERSION - same here
> 
> Maybe ptxdist could offer using one of those as a fake timestamp, if the
> appropriate format is used.

I'd expect that these change as well.

Hmmm, maybe make it configurable
- ptxdist release: what we have now
- toolchain release: if the compiler changes, then the result will be
  different anyways.
- manual 'date' string: any use-case for this?

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Alexander Dahl]

[-- Attachment #1.1: Type: text/plain, Size: 1014 bytes --]

Hello Florian,

On Tue, Oct 16, 2018 at 10:57:20AM +0000, Baeuerle, Florian wrote:
> SOURCE_DATE_EPOCH uses the ptxdist release number for the timestamp, thus the
> timestamp changes with every ptxdist release. I'd prefer to *really* get the
> same binary when the source code was not changed at all. A ptxdist version bump
> has a high potential of not touching anything barebox related.
> 
> I also do not like mangling the barebox version into a timestamp, but I couldn't
> come up with any better idea.

If I understood correctly, you want some timestamp which does not
change, as long as barebox is not touched. Why not base it on the
barebox version then? What would be the problem with that?

Greets
Alex

-- 
/"\ ASCII RIBBON | »With the first link, the chain is forged. The first
\ / CAMPAIGN     | speech censured, the first thought forbidden, the
 X  AGAINST      | first freedom denied, chains us all irrevocably.«
/ \ HTML MAIL    | (Jean-Luc Picard, quoting Judge Aaron Satie)

[-- Attachment #1.2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[-- Attachment #2: Type: text/plain, Size: 91 bytes --]

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/3] barebox: add option for building barebox reproducibly

[Michael Olbrich]

On Tue, Oct 16, 2018 at 07:15:37PM +0200, Alexander Dahl wrote:
> On Tue, Oct 16, 2018 at 10:57:20AM +0000, Baeuerle, Florian wrote:
> > SOURCE_DATE_EPOCH uses the ptxdist release number for the timestamp, thus the
> > timestamp changes with every ptxdist release. I'd prefer to *really* get the
> > same binary when the source code was not changed at all. A ptxdist version bump
> > has a high potential of not touching anything barebox related.
> > 
> > I also do not like mangling the barebox version into a timestamp, but I couldn't
> > come up with any better idea.
> 
> If I understood correctly, you want some timestamp which does not
> change, as long as barebox is not touched. Why not base it on the
> barebox version then? What would be the problem with that?

That's what his patch does. I'd like to have one timestamp for all of
PTXdist.

Michael


-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 3/3] barebox_mlo: add option for building barebox_mlo reproducibly

[Baeuerle, Florian]

Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
---
 platforms/barebox_mlo.in | 11 +++++++++++
 rules/barebox_mlo.make   | 29 ++++++++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/platforms/barebox_mlo.in b/platforms/barebox_mlo.in
index 01c8669bc..7c3266d0b 100644
--- a/platforms/barebox_mlo.in
+++ b/platforms/barebox_mlo.in
@@ -1,6 +1,7 @@
 ## SECTION=bootloader
 
 menuconfig BAREBOX_MLO
+	select HOST_FAKETIME if BAREBOX_MLO_REPRODUCIBLE
 	prompt "barebox mlo                   "
 	bool
 	help
@@ -33,4 +34,14 @@ config BAREBOX_MLO_CONFIG
 	help
 	  This entry specifies the .config file used to compile mlo.
 
+config BAREBOX_MLO_REPRODUCIBLE
+	prompt "build reproducibly"
+	bool
+	default BAREBOX_REPRODUCIBLE		if BAREBOX
+	default n				if !BAREBOX
+	help
+	  With this, fake timestamps are injected to the barebox build, so the
+	  resulting barebox images are identical when the sources haven't been
+	  changed.
+
 endif
diff --git a/rules/barebox_mlo.make b/rules/barebox_mlo.make
index 368da0e8f..8c0c07664 100644
--- a/rules/barebox_mlo.make
+++ b/rules/barebox_mlo.make
@@ -38,8 +38,25 @@ BAREBOX_MLO_WRAPPER_BLACKLIST := \
 	TARGET_DEBUG \
 	TARGET_BUILD_ID
 
+BAREBOX_MLO_ENV	:= \
+	KCONFIG_NOTIMESTAMP=1 \
+	pkg_wrapper_blacklist="$(BAREBOX_MLO_WRAPPER_BLACKLIST)"
+
+ifdef PTXCONF_BAREBOX_MLO_REPRODUCIBLE
+BAREBOX_MLO_FAKE_TIMESTAMP := \
+	$(shell echo $(BAREBOX_MLO_VERSION) | \
+		sed -E "s/([0-9]{4})\.([0-9]{1,2}).*/\1-\2-01 00:00:00/")
+BAREBOX_MLO_FAKETIME := faketime -f "$(BAREBOX_MLO_FAKE_TIMESTAMP)"
+BAREBOX_MLO_ENV := $(BAREBOX_MLO_ENV) \
+	KBUILD_BUILD_TIMESTAMP="$(BAREBOX_MLO_FAKE_TIMESTAMP)" \
+	KBUILD_BUILD_VERSION="0" \
+	KBUILD_BUILD_USER="ptxdist" \
+	KBUILD_BUILD_HOST="ptxdist"
+else
+BAREBOX_MLO_FAKETIME :=
+endif
+
 BAREBOX_MLO_PATH	:= PATH=$(CROSS_PATH)
-BAREBOX_MLO_CONF_ENV	:= KCONFIG_NOTIMESTAMP=1
 BAREBOX_MLO_CONF_TOOL	:= kconfig
 BAREBOX_MLO_CONF_OPT	:= \
 			V=$(PTXDIST_VERBOSE) \
@@ -62,6 +79,16 @@ endif
 
 $(STATEDIR)/barebox_mlo.prepare: $(BAREBOX_MLO_CONFIG)
 
+# ----------------------------------------------------------------------------
+# Compile
+# ----------------------------------------------------------------------------
+
+$(STATEDIR)/barebox_mlo.compile:
+	@$(call targetinfo)
+	cd $(BAREBOX_MLO_DIR) && $(BAREBOX_MLO_PATH) $(BAREBOX_MLO_ENV) \
+		$(BAREBOX_MLO_FAKETIME) $(MAKE) $(BAREBOX_MLO_MAKE_OPT)
+	@$(call touch)
+
 # ----------------------------------------------------------------------------
 # Install
 # ----------------------------------------------------------------------------
-- 
2.19.0

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Roland Hieber]

Hmmm, we had a similar patch in August that added libfaketime as a
runtime dependency to PTXdist in order to detect changes to packages'
patch series:

https://www.mail-archive.com/ptxdist@pengutronix.de/msg13198.html

If we can be sure that faketime is present at run time, we could leave
it out as a host package (or maybe only need a host-system-faketime
package instead).

Jon (or mol?), I think the patches from that series were never applied,
but in light of recent development I don't know if they are still
needed. In any case, does it make more sense to have faketime as a host
system dependency available at PTXdist startup, or is a host-* package
in PTXdist as proposed by Florian here sufficient?

 - Roland

On Thu, Oct 11, 2018 at 02:19:15PM +0000, Baeuerle, Florian wrote:
> Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> ---
>  rules/host-faketime.in   | 10 +++++++++
>  rules/host-faketime.make | 47 ++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 57 insertions(+)
>  create mode 100644 rules/host-faketime.in
>  create mode 100644 rules/host-faketime.make
> 
> diff --git a/rules/host-faketime.in b/rules/host-faketime.in
> new file mode 100644
> index 000000000..1a2e7ae19
> --- /dev/null
> +++ b/rules/host-faketime.in
> @@ -0,0 +1,10 @@
> +## SECTION=hosttools_noprompt
> +## SECTION=hosttools_platform
> +
> +config HOST_FAKETIME
> +	tristate
> +	default ALLYES
> +	help
> +	  libfaketime intercepts various system calls that programs use to
> +	  retrieve the current date and time. It then reports modified (faked)
> +	  dates and times to these programs.
> diff --git a/rules/host-faketime.make b/rules/host-faketime.make
> new file mode 100644
> index 000000000..462854eb9
> --- /dev/null
> +++ b/rules/host-faketime.make
> @@ -0,0 +1,47 @@
> +# -*-makefile-*-
> +#
> +# Copyright (C) 2018 by Florian Bäuerle <florian.baeuerle@allegion.com>
> +#
> +# See CREDITS for details about who has contributed to this project.
> +#
> +# For further information about the PTXdist project and license conditions
> +# see the README file.
> +#
> +
> +#
> +# We provide this package
> +#
> +HOST_PACKAGES-$(PTXCONF_HOST_FAKETIME) += host-faketime
> +
> +#
> +# Paths and names
> +#
> +HOST_FAKETIME_VERSION	= $(LIBFAKETIME_VERSION)
> +HOST_FAKETIME_MD5	= $(LIBFAKETIME_MD5)
> +HOST_FAKETIME		= $(LIBFAKETIME)
> +HOST_FAKETIME_SUFFIX	= $(LIBFAKETIME_SUFFIX)
> +HOST_FAKETIME_URL	= $(LIBFAKETIME_URL)
> +HOST_FAKETIME_SOURCE	= $(LIBFAKETIME_SOURCE)
> +HOST_FAKETIME_LICENSE	= $(LIBFAKETIME_LICENSE)
> +HOST_FAKETIME_DIR	= $(HOST_BUILDDIR)/$(HOST_FAKETIME)
> +
> +# ----------------------------------------------------------------------------
> +# Prepare
> +# ----------------------------------------------------------------------------
> +
> +HOST_FAKETIME_CONF_TOOL	= NO
> +
> +# ----------------------------------------------------------------------------
> +# Compile
> +# ----------------------------------------------------------------------------
> +
> +HOST_FAKETIME_MAKE_ENV	= $(HOST_ENV)
> +HOST_FAKETIME_MAKE_OPT	= PREFIX=$(PTXCONF_SYSROOT_HOST) DESTDIR=
> +
> +# ----------------------------------------------------------------------------
> +# Install
> +# ----------------------------------------------------------------------------
> +
> +HOST_FAKETIME_INSTALL_OPT	= PREFIX=$(PTXCONF_SYSROOT_HOST) DESTDIR= install
> +
> +# vim: syntax=make
> -- 
> 2.19.0
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de

-- 
Roland Hieber                     | r.hieber@pengutronix.de     |
Pengutronix e.K.                  | https://www.pengutronix.de/ |
Peiner Str. 6-8, 31137 Hildesheim | Phone: +49-5121-206917-5086 |
Amtsgericht Hildesheim, HRA 2686  | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Jon Ringle]

On Thu, Oct 11, 2018 at 10:43 AM Roland Hieber <r.hieber@pengutronix.de> wrote:
>
> Hmmm, we had a similar patch in August that added libfaketime as a
> runtime dependency to PTXdist in order to detect changes to packages'
> patch series:
>
> https://www.mail-archive.com/ptxdist@pengutronix.de/msg13198.html
>
> If we can be sure that faketime is present at run time, we could leave
> it out as a host package (or maybe only need a host-system-faketime
> package instead).
>
> Jon (or mol?), I think the patches from that series were never applied,
> but in light of recent development I don't know if they are still
> needed. In any case, does it make more sense to have faketime as a host
> system dependency available at PTXdist startup, or is a host-* package
> in PTXdist as proposed by Florian here sufficient?


mol had suggested the following as an alternative to my patch, but I
haven't gotten around to trying it:
> I think it should be possible to provide the correct timestamps with
> GIT_AUTHOR_DATE and/or GIT_COMMITTER_DATE or something like that.

In any case, what I proposed was independent of having a host faketime.

>
>  - Roland
>
> On Thu, Oct 11, 2018 at 02:19:15PM +0000, Baeuerle, Florian wrote:
> > Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> > ---
> >  rules/host-faketime.in   | 10 +++++++++
> >  rules/host-faketime.make | 47 ++++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 57 insertions(+)
> >  create mode 100644 rules/host-faketime.in
> >  create mode 100644 rules/host-faketime.make
> >
> > diff --git a/rules/host-faketime.in b/rules/host-faketime.in
> > new file mode 100644
> > index 000000000..1a2e7ae19
> > --- /dev/null
> > +++ b/rules/host-faketime.in
> > @@ -0,0 +1,10 @@
> > +## SECTION=hosttools_noprompt
> > +## SECTION=hosttools_platform
> > +
> > +config HOST_FAKETIME
> > +     tristate
> > +     default ALLYES
> > +     help
> > +       libfaketime intercepts various system calls that programs use to
> > +       retrieve the current date and time. It then reports modified (faked)
> > +       dates and times to these programs.
> > diff --git a/rules/host-faketime.make b/rules/host-faketime.make
> > new file mode 100644
> > index 000000000..462854eb9
> > --- /dev/null
> > +++ b/rules/host-faketime.make
> > @@ -0,0 +1,47 @@
> > +# -*-makefile-*-
> > +#
> > +# Copyright (C) 2018 by Florian Bäuerle <florian.baeuerle@allegion.com>
> > +#
> > +# See CREDITS for details about who has contributed to this project.
> > +#
> > +# For further information about the PTXdist project and license conditions
> > +# see the README file.
> > +#
> > +
> > +#
> > +# We provide this package
> > +#
> > +HOST_PACKAGES-$(PTXCONF_HOST_FAKETIME) += host-faketime
> > +
> > +#
> > +# Paths and names
> > +#
> > +HOST_FAKETIME_VERSION        = $(LIBFAKETIME_VERSION)
> > +HOST_FAKETIME_MD5    = $(LIBFAKETIME_MD5)
> > +HOST_FAKETIME                = $(LIBFAKETIME)
> > +HOST_FAKETIME_SUFFIX = $(LIBFAKETIME_SUFFIX)
> > +HOST_FAKETIME_URL    = $(LIBFAKETIME_URL)
> > +HOST_FAKETIME_SOURCE = $(LIBFAKETIME_SOURCE)
> > +HOST_FAKETIME_LICENSE        = $(LIBFAKETIME_LICENSE)
> > +HOST_FAKETIME_DIR    = $(HOST_BUILDDIR)/$(HOST_FAKETIME)

I would suggest that it should be called host-libfaketime, which
should then allow much of the boilerplate code above to be removed

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Baeuerle, Florian]

Hi,

Am Donnerstag, den 11.10.2018, 17:30 -0400 schrieb Jon Ringle:
> > > +#
> > > +HOST_FAKETIME_VERSION        = $(LIBFAKETIME_VERSION)
> > > +HOST_FAKETIME_MD5    = $(LIBFAKETIME_MD5)
> > > +HOST_FAKETIME                = $(LIBFAKETIME)
> > > +HOST_FAKETIME_SUFFIX = $(LIBFAKETIME_SUFFIX)
> > > +HOST_FAKETIME_URL    = $(LIBFAKETIME_URL)
> > > +HOST_FAKETIME_SOURCE = $(LIBFAKETIME_SOURCE)
> > > +HOST_FAKETIME_LICENSE        = $(LIBFAKETIME_LICENSE)
> > > +HOST_FAKETIME_DIR    = $(HOST_BUILDDIR)/$(HOST_FAKETIME)
> 
> I would suggest that it should be called host-libfaketime, which
> should then allow much of the boilerplate code above to be removed

I tried that first, but probably I did something wrong – ptxdist did not
download the sources.

I'll give it another try.


- Florian
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Michael Olbrich]

On Tue, Oct 16, 2018 at 07:33:48AM +0000, Baeuerle, Florian wrote:
> Hi,
> 
> Am Donnerstag, den 11.10.2018, 17:30 -0400 schrieb Jon Ringle:
> > > > +#
> > > > +HOST_FAKETIME_VERSION        = $(LIBFAKETIME_VERSION)
> > > > +HOST_FAKETIME_MD5    = $(LIBFAKETIME_MD5)
> > > > +HOST_FAKETIME                = $(LIBFAKETIME)
> > > > +HOST_FAKETIME_SUFFIX = $(LIBFAKETIME_SUFFIX)
> > > > +HOST_FAKETIME_URL    = $(LIBFAKETIME_URL)
> > > > +HOST_FAKETIME_SOURCE = $(LIBFAKETIME_SOURCE)
> > > > +HOST_FAKETIME_LICENSE        = $(LIBFAKETIME_LICENSE)
> > > > +HOST_FAKETIME_DIR    = $(HOST_BUILDDIR)/$(HOST_FAKETIME)
> > 
> > I would suggest that it should be called host-libfaketime, which
> > should then allow much of the boilerplate code above to be removed
> 
> I tried that first, but probably I did something wrong – ptxdist did not
> download the sources.
> 
> I'll give it another try.

There is a bug in the latest release. If the host package is selected but
not the target package, then downloading fails. Try this with master or an
older release.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Baeuerle, Florian]

Hi Roland,

Am Donnerstag, den 11.10.2018, 16:43 +0200 schrieb Roland Hieber:
> Hmmm, we had a similar patch in August that added libfaketime as a
> runtime dependency to PTXdist in order to detect changes to packages'
> patch series:
> 
> https://www.mail-archive.com/ptxdist@pengutronix.de/msg13198.html
> 
> If we can be sure that faketime is present at run time, we could leave
> it out as a host package (or maybe only need a host-system-faketime
> package instead).

Is there also a mechanism in ptxdist that allows leaving out  system packages if
the provided tools are already present?

I did not want to introduce a hard dependency for ptxdist (i. e. host-system-
faketime package), because libfaketime does not seem to be a first class citizen
in some distributions (community repo in archlinux, epel in centos).


- Florian
_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Michael Olbrich]

On Fri, Oct 12, 2018 at 07:47:35AM +0000, Baeuerle, Florian wrote:
> Am Donnerstag, den 11.10.2018, 16:43 +0200 schrieb Roland Hieber:
> > Hmmm, we had a similar patch in August that added libfaketime as a
> > runtime dependency to PTXdist in order to detect changes to packages'
> > patch series:
> > 
> > https://www.mail-archive.com/ptxdist@pengutronix.de/msg13198.html
> > 
> > If we can be sure that faketime is present at run time, we could leave
> > it out as a host package (or maybe only need a host-system-faketime
> > package instead).
> 
> Is there also a mechanism in ptxdist that allows leaving out  system packages if
> the provided tools are already present?

No, and we prefer to keep things working for a long time, so minimizing
external dependencies is important.
So, host-system-* is for big stuff, like perl/python/... or really stable
things like bc.

> I did not want to introduce a hard dependency for ptxdist (i. e. host-system-
> faketime package), because libfaketime does not seem to be a first class citizen
> in some distributions (community repo in archlinux, epel in centos).

I think libfaketime is small enough that building it should not be
problematic.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Alexander Dahl]

Hello,

Am Dienstag, 16. Oktober 2018, 10:51:04 CEST schrieb Michael Olbrich:
> On Fri, Oct 12, 2018 at 07:47:35AM +0000, Baeuerle, Florian wrote:
> > Is there also a mechanism in ptxdist that allows leaving out  system
> > packages if the provided tools are already present?
> 
> No, and we prefer to keep things working for a long time, so minimizing
> external dependencies is important.
> So, host-system-* is for big stuff, like perl/python/... or really stable
> things like bc.

What about CMake? That takes quite a lot of time to build.

% time p -q -j15 install host-cmake
started : host-cmake.4b2327f4c98f307e852ac59967b0fbba.cfghash
finished: host-cmake.4b2327f4c98f307e852ac59967b0fbba.cfghash
started : host-cmake.get
finished: host-cmake.get
started : host-cmake.extract
finished: host-cmake.extract
started : host-cmake.extract.post
finished: host-cmake.extract.post
started : host-cmake.prepare
finished: host-cmake.prepare
started : host-cmake.compile
finished: host-cmake.compile
started : host-cmake.install
finished: host-cmake.install
started : host-cmake.install.pack
finished: host-cmake.install.pack
started : host-cmake.install.post
finished: host-cmake.install.post
ptxdist -q -j15 install host-cmake  320,24s user 39,07s system 203% cpu 
2:56,40 total

That's more than 5 minutes!

Greets
Alex


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Michael Olbrich]

Hi,

On Tue, Oct 16, 2018 at 01:34:15PM +0200, Alexander Dahl wrote:
> Am Dienstag, 16. Oktober 2018, 10:51:04 CEST schrieb Michael Olbrich:
> > On Fri, Oct 12, 2018 at 07:47:35AM +0000, Baeuerle, Florian wrote:
> > > Is there also a mechanism in ptxdist that allows leaving out  system
> > > packages if the provided tools are already present?
> > 
> > No, and we prefer to keep things working for a long time, so minimizing
> > external dependencies is important.
> > So, host-system-* is for big stuff, like perl/python/... or really stable
> > things like bc.
> 
> What about CMake? That takes quite a lot of time to build.
> 
> % time p -q -j15 install host-cmake
[...]
> That's more than 5 minutes!

The problem is, that system cmake is sometimes too old. It's not so bad
these days but it can be a problem.

Michael

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 1/3] host-faketime: new package

[Michael Olbrich]

On Thu, Oct 11, 2018 at 02:19:15PM +0000, Baeuerle, Florian wrote:
> Signed-off-by: Florian Bäuerle <florian.baeuerle@allegion.com>
> ---
>  rules/host-faketime.in   | 10 +++++++++
>  rules/host-faketime.make | 47 ++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 57 insertions(+)
>  create mode 100644 rules/host-faketime.in
>  create mode 100644 rules/host-faketime.make
> 
> diff --git a/rules/host-faketime.in b/rules/host-faketime.in
> new file mode 100644
> index 000000000..1a2e7ae19
> --- /dev/null
> +++ b/rules/host-faketime.in
> @@ -0,0 +1,10 @@
> +## SECTION=hosttools_noprompt
> +## SECTION=hosttools_platform
> +
> +config HOST_FAKETIME
> +	tristate
> +	default ALLYES
> +	help
> +	  libfaketime intercepts various system calls that programs use to
> +	  retrieve the current date and time. It then reports modified (faked)
> +	  dates and times to these programs.
> diff --git a/rules/host-faketime.make b/rules/host-faketime.make
> new file mode 100644
> index 000000000..462854eb9
> --- /dev/null
> +++ b/rules/host-faketime.make
> @@ -0,0 +1,47 @@
> +# -*-makefile-*-
> +#
> +# Copyright (C) 2018 by Florian Bäuerle <florian.baeuerle@allegion.com>
> +#
> +# See CREDITS for details about who has contributed to this project.
> +#
> +# For further information about the PTXdist project and license conditions
> +# see the README file.
> +#
> +
> +#
> +# We provide this package
> +#
> +HOST_PACKAGES-$(PTXCONF_HOST_FAKETIME) += host-faketime
> +
> +#
> +# Paths and names
> +#
> +HOST_FAKETIME_VERSION	= $(LIBFAKETIME_VERSION)
> +HOST_FAKETIME_MD5	= $(LIBFAKETIME_MD5)
> +HOST_FAKETIME		= $(LIBFAKETIME)
> +HOST_FAKETIME_SUFFIX	= $(LIBFAKETIME_SUFFIX)
> +HOST_FAKETIME_URL	= $(LIBFAKETIME_URL)
> +HOST_FAKETIME_SOURCE	= $(LIBFAKETIME_SOURCE)
> +HOST_FAKETIME_LICENSE	= $(LIBFAKETIME_LICENSE)
> +HOST_FAKETIME_DIR	= $(HOST_BUILDDIR)/$(HOST_FAKETIME)
> +
> +# ----------------------------------------------------------------------------
> +# Prepare
> +# ----------------------------------------------------------------------------
> +
> +HOST_FAKETIME_CONF_TOOL	= NO
> +
> +# ----------------------------------------------------------------------------
> +# Compile
> +# ----------------------------------------------------------------------------
> +
> +HOST_FAKETIME_MAKE_ENV	= $(HOST_ENV)
> +HOST_FAKETIME_MAKE_OPT	= PREFIX=$(PTXCONF_SYSROOT_HOST) DESTDIR=

Why is this needed? I'd prefer to have things relocatable.
This works like fakeroot, right? Maybe you can do something like
patches/fakeroot-1.21/0100-make-fakeroot-relocatable.patch

Michael

> +
> +# ----------------------------------------------------------------------------
> +# Install
> +# ----------------------------------------------------------------------------
> +
> +HOST_FAKETIME_INSTALL_OPT	= PREFIX=$(PTXCONF_SYSROOT_HOST) DESTDIR= install
> +
> +# vim: syntax=make
> -- 
> 2.19.0
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de

-- 
Pengutronix e.K.                           |                             |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de