[ptxdist] [PATCH 1/2] ptxdist: add licensecheck command

[ptxdist] [PATCH 1/2] ptxdist: add licensecheck command

[Roland Hieber]

Make it possible to check all MD5 sums of license files by building all
report stages without also building the license report (which produces
tons of LaTeX output).

Signed-off-by: Roland Hieber <rhi@pengutronix.de>
---
 bin/ptxdist                              | 6 ++++++
 doc/ref_parameter.inc                    | 3 +++
 rules/post/ptxd_make_license_report.make | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/bin/ptxdist b/bin/ptxdist
index 1e10c08d47f2..4a372ff6392c 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -752,6 +752,7 @@ Misc:
 				it is known by "make"
   printnext <var>		assumes that the contents of <var> is another
 				variable and print the contents of this variable
+  licensecheck			check md5sums of license files for all packages
   list-packages			print a list of all selected packages
   local-src <pkg> [<directory>]	overwrite a package source with a locally provided
 				directory containing the sourcecode.
@@ -1776,6 +1777,11 @@ EOF
 			ptxd_make_log "${images[@]}"
 			exit
 			;;
+		licensecheck)
+			check_premake_compiler &&
+			ptxd_make_log report
+			exit
+			;;
 		list-packages)
 			check_config &&
 			check_deps || return
diff --git a/doc/ref_parameter.inc b/doc/ref_parameter.inc
index 74689b9d3cff..aaac8956f147 100644
--- a/doc/ref_parameter.inc
+++ b/doc/ref_parameter.inc
@@ -234,6 +234,9 @@ Misc Actions
   contents of this variable. Shell variables are currently not checked here.
   All other rules for ``print`` apply.
 
+``licensecheck``
+  For all selected packages, check the MD5 sums of license files.
+
 ``list-packages``
   print a list of all selected packages. This list does not include the
   host and cross tools.
diff --git a/rules/post/ptxd_make_license_report.make b/rules/post/ptxd_make_license_report.make
index 47dc176668d0..1a13c3ad4964 100644
--- a/rules/post/ptxd_make_license_report.make
+++ b/rules/post/ptxd_make_license_report.make
@@ -15,6 +15,8 @@ PTX_PACKAGES_TARGET := \
 	$(PTX_PACKAGES_INSTALL) \
 	$(EXTRA_PACKAGES)
 
+report: $(addprefix $(STATEDIR)/,$(addsuffix .report,$(PTX_PACKAGES_SELECTED)))
+
 PHONY += license-report
 
 license-report: \
-- 
2.26.2


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH 2/2] ptxd_lib_dgen: optimize ordering only when packages are being fully built

[Roland Hieber]

There are use cases where -j is useful to have on the command line to
speed up runs with large amount of packages, but when targetinstall
stages are not needed at all. For example:

    ptxdist -j license-check
    ptxdist -j make license-report
    ptxdist -j extract <long list of package names>

Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
to extract") introduced better interleaving of extract stages to
optimise IO load during full builds. This also had the effect that in
the above cases all targetinstall stages of dependencies were built too,
even when they are not needed at all. Improve that use case by adding a
flag in the environment and only set it in case of 'ptxdist go',
'ptxdist images' or 'ptxdist image <image-name>'.

Signed-off-by: Roland Hieber <rhi@pengutronix.de>
---
 bin/ptxdist                   | 3 +++
 scripts/lib/ptxd_lib_dgen.awk | 5 +++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/ptxdist b/bin/ptxdist
index 4a372ff6392c..6034d170fcdc 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -1764,10 +1764,12 @@ EOF
 			exit
 			;;
 		go|images)
+			PTXDIST_OPTIMIZE_IO=true
 			do_${cmd}
 			exit
 			;;
 		image)
+			PTXDIST_OPTIMIZE_IO=true
 			if [ ${#} -eq 0 ]; then
 				echo "No image given."
 				exit 1
@@ -2564,6 +2566,7 @@ setup_export() {
 		PTXDIST_ICECC_CREATE_ENV \
 		PTXDIST_LOG_PROMPT \
 		PTXDIST_OUTPUT_SYNC \
+		PTXDIST_OPTIMIZE_IO \
 		PTXDIST_DIRTY \
 		PTXDIST_FORCE \
 		PTXDIST_PEDANTIC \
diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
index 8ee88827b1f8..8c09c531129e 100644
--- a/scripts/lib/ptxd_lib_dgen.awk
+++ b/scripts/lib/ptxd_lib_dgen.awk
@@ -17,6 +17,7 @@ BEGIN {
 	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
 	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
 	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
+	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
 	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
 	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
 	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
@@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
 	for (i = 1; i <= n; i++) {
 		this_dep = PKG_to_pkg[this_DEP_array[i]]
 
-		if (PARALLEL != "-j1")
+		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
 			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
 		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
 		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
@@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
 		else
 			virtual = "base";
 	}
-	if (PARALLEL != "-j1")
+	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
 		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
-- 
2.26.2


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/2] ptxd_lib_dgen: optimize ordering only when packages are being fully built

[Roland Hieber]

On Mon, May 25, 2020 at 11:28:20AM +0200, Roland Hieber wrote:
> There are use cases where -j is useful to have on the command line to
> speed up runs with large amount of packages, but when targetinstall
> stages are not needed at all. For example:
> 
>     ptxdist -j license-check
>     ptxdist -j make license-report
>     ptxdist -j extract <long list of package names>
> 
> Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
> to extract") introduced better interleaving of extract stages to
> optimise IO load during full builds. This also had the effect that in

gah, optimise... optimize... one reason why I had to debug my own patch
for one hour this morning... ^^ (well, not important here :))

 - Roland

> the above cases all targetinstall stages of dependencies were built too,
> even when they are not needed at all. Improve that use case by adding a
> flag in the environment and only set it in case of 'ptxdist go',
> 'ptxdist images' or 'ptxdist image <image-name>'.
> 
> Signed-off-by: Roland Hieber <rhi@pengutronix.de>
> ---
>  bin/ptxdist                   | 3 +++
>  scripts/lib/ptxd_lib_dgen.awk | 5 +++--
>  2 files changed, 6 insertions(+), 2 deletions(-)
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 4a372ff6392c..6034d170fcdc 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -1764,10 +1764,12 @@ EOF
>  			exit
>  			;;
>  		go|images)
> +			PTXDIST_OPTIMIZE_IO=true
>  			do_${cmd}
>  			exit
>  			;;
>  		image)
> +			PTXDIST_OPTIMIZE_IO=true
>  			if [ ${#} -eq 0 ]; then
>  				echo "No image given."
>  				exit 1
> @@ -2564,6 +2566,7 @@ setup_export() {
>  		PTXDIST_ICECC_CREATE_ENV \
>  		PTXDIST_LOG_PROMPT \
>  		PTXDIST_OUTPUT_SYNC \
> +		PTXDIST_OPTIMIZE_IO \
>  		PTXDIST_DIRTY \
>  		PTXDIST_FORCE \
>  		PTXDIST_PEDANTIC \
> diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
> index 8ee88827b1f8..8c09c531129e 100644
> --- a/scripts/lib/ptxd_lib_dgen.awk
> +++ b/scripts/lib/ptxd_lib_dgen.awk
> @@ -17,6 +17,7 @@ BEGIN {
>  	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
>  	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
>  	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
> +	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
>  	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
>  	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
>  	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
> @@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
>  	for (i = 1; i <= n; i++) {
>  		this_dep = PKG_to_pkg[this_DEP_array[i]]
>  
> -		if (PARALLEL != "-j1")
> +		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
> @@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
>  		else
>  			virtual = "base";
>  	}
> -	if (PARALLEL != "-j1")
> +	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
> -- 
> 2.26.2
> 
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de
> 

-- 
Roland Hieber, Pengutronix e.K.          | r.hieber@pengutronix.de     |
Steuerwalder Str. 21                     | https://www.pengutronix.de/ |
31137 Hildesheim, Germany                | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686         | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [PATCH 2/2] ptxd_lib_dgen: optimize ordering only when packages are being fully built

[Michael Olbrich]

On Mon, May 25, 2020 at 11:28:20AM +0200, Roland Hieber wrote:
> There are use cases where -j is useful to have on the command line to
> speed up runs with large amount of packages, but when targetinstall
> stages are not needed at all. For example:
> 
>     ptxdist -j license-check
>     ptxdist -j make license-report
>     ptxdist -j extract <long list of package names>
> 
> Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
> to extract") introduced better interleaving of extract stages to
> optimise IO load during full builds. This also had the effect that in
> the above cases all targetinstall stages of dependencies were built too,
> even when they are not needed at all. Improve that use case by adding a
> flag in the environment and only set it in case of 'ptxdist go',
> 'ptxdist images' or 'ptxdist image <image-name>'.
> 
> Signed-off-by: Roland Hieber <rhi@pengutronix.de>
> ---
>  bin/ptxdist                   | 3 +++
>  scripts/lib/ptxd_lib_dgen.awk | 5 +++--
>  2 files changed, 6 insertions(+), 2 deletions(-)
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 4a372ff6392c..6034d170fcdc 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -1764,10 +1764,12 @@ EOF
>  			exit
>  			;;
>  		go|images)
> +			PTXDIST_OPTIMIZE_IO=true
>  			do_${cmd}
>  			exit
>  			;;
>  		image)
> +			PTXDIST_OPTIMIZE_IO=true

Hmmm, I think anything, that includes prepare stages should get this flag.
Or maybe reverse the logic and flag commands where this should be skipped.
I don't know which would be easier.

Michael

>  			if [ ${#} -eq 0 ]; then
>  				echo "No image given."
>  				exit 1
> @@ -2564,6 +2566,7 @@ setup_export() {
>  		PTXDIST_ICECC_CREATE_ENV \
>  		PTXDIST_LOG_PROMPT \
>  		PTXDIST_OUTPUT_SYNC \
> +		PTXDIST_OPTIMIZE_IO \
>  		PTXDIST_DIRTY \
>  		PTXDIST_FORCE \
>  		PTXDIST_PEDANTIC \
> diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
> index 8ee88827b1f8..8c09c531129e 100644
> --- a/scripts/lib/ptxd_lib_dgen.awk
> +++ b/scripts/lib/ptxd_lib_dgen.awk
> @@ -17,6 +17,7 @@ BEGIN {
>  	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
>  	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
>  	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
> +	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
>  	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
>  	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
>  	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
> @@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
>  	for (i = 1; i <= n; i++) {
>  		this_dep = PKG_to_pkg[this_DEP_array[i]]
>  
> -		if (PARALLEL != "-j1")
> +		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
> @@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
>  		else
>  			virtual = "base";
>  	}
> -	if (PARALLEL != "-j1")
> +	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
> -- 
> 2.26.2
> 
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de
> 

-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

[ptxdist] [PATCH v2 2/2] ptxd_lib_dgen: optimize ordering only when dependencies are needed

[Roland Hieber]

There are use cases where -j is useful to have on the command line to
speed up runs with large amount of packages, but when the install stages
are not needed at all. For example:

    ptxdist -j license-check
    ptxdist -j make license-report
    ptxdist -j extract <long list of package names>

Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
to extract") introduced better interleaving of extract stages to
optimize IO load during full builds. This also had the effect that in
the above cases all install stages of dependencies were built too, even
when they are not needed at all.

Improve those use cases by only optimizing the extract stage ordering if
at least the prepare stage(s) of the packages being requested are needed
(since the prepare stage depends on the install stages of dependencies,
those will be needed anyway).

Signed-off-by: Roland Hieber <rhi@pengutronix.de>
---
v2: also set PTXDIST_OPTIMIZE_IO in prepare, compile, install,
    targetinstall, tags, nfsroot; and adapt log message accordingly

On Mon, May 25, 2020 at 02:47:33PM +0200, Michael Olbrich wrote:
> Hmmm, I think anything, that includes prepare stages should get this flag.
> Or maybe reverse the logic and flag commands where this should be skipped.
> I don't know which would be easier.
> 
> Michael

Like this?
A positive list is still easier :)

 - Roland

---
 bin/ptxdist                   | 9 ++++++++-
 scripts/lib/ptxd_lib_dgen.awk | 5 +++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/bin/ptxdist b/bin/ptxdist
index 4a372ff6392c..0d85c1290868 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -1579,7 +1579,10 @@ parse_second()
 
 ######## standard target, directly into make
 
-		extract|prepare|compile|install|targetinstall|tags)
+		prepare|compile|install|targetinstall|tags)
+			PTXDIST_OPTIMIZE_IO=true
+			;&  # fallthrough
+		extract)
 			local cmd_post
 			declare -a pkgs
 
@@ -1764,10 +1767,12 @@ EOF
 			exit
 			;;
 		go|images)
+			PTXDIST_OPTIMIZE_IO=true
 			do_${cmd}
 			exit
 			;;
 		image)
+			PTXDIST_OPTIMIZE_IO=true
 			if [ ${#} -eq 0 ]; then
 				echo "No image given."
 				exit 1
@@ -1823,6 +1828,7 @@ EOF
 			exit
 			;;
 		nfsroot)
+			PTXDIST_OPTIMIZE_IO=true
 			check_premake_compiler &&
 			ptxd_make_log ptxd_make_nfsd
 			exit
@@ -2564,6 +2570,7 @@ setup_export() {
 		PTXDIST_ICECC_CREATE_ENV \
 		PTXDIST_LOG_PROMPT \
 		PTXDIST_OUTPUT_SYNC \
+		PTXDIST_OPTIMIZE_IO \
 		PTXDIST_DIRTY \
 		PTXDIST_FORCE \
 		PTXDIST_PEDANTIC \
diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
index 8ee88827b1f8..8c09c531129e 100644
--- a/scripts/lib/ptxd_lib_dgen.awk
+++ b/scripts/lib/ptxd_lib_dgen.awk
@@ -17,6 +17,7 @@ BEGIN {
 	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
 	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
 	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
+	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
 	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
 	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
 	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
@@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
 	for (i = 1; i <= n; i++) {
 		this_dep = PKG_to_pkg[this_DEP_array[i]]
 
-		if (PARALLEL != "-j1")
+		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
 			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
 		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
 		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
@@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
 		else
 			virtual = "base";
 	}
-	if (PARALLEL != "-j1")
+	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
 		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
-- 
2.26.2


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de

Re: [ptxdist] [APPLIED] ptxd_lib_dgen: optimize ordering only when dependencies are needed

[Michael Olbrich]

Thanks, applied as 943ca0008b5add0d921153fb9ba365fcc33a8318.

Michael

[sent from post-receive hook]

On Fri, 05 Jun 2020 07:59:07 +0200, Roland Hieber <rhi@pengutronix.de> wrote:
> There are use cases where -j is useful to have on the command line to
> speed up runs with large amount of packages, but when the install stages
> are not needed at all. For example:
> 
>     ptxdist -j license-check
>     ptxdist -j make license-report
>     ptxdist -j extract <long list of package names>
> 
> Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
> to extract") introduced better interleaving of extract stages to
> optimize IO load during full builds. This also had the effect that in
> the above cases all install stages of dependencies were built too, even
> when they are not needed at all.
> 
> Improve those use cases by only optimizing the extract stage ordering if
> at least the prepare stage(s) of the packages being requested are needed
> (since the prepare stage depends on the install stages of dependencies,
> those will be needed anyway).
> 
> Signed-off-by: Roland Hieber <rhi@pengutronix.de>
> Message-Id: <20200525202209.24858-1-rhi@pengutronix.de>
> Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 6635a96a1cf5..6b789d9a73fa 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -1579,7 +1579,10 @@ parse_second()
>  
>  ######## standard target, directly into make
>  
> -		extract|prepare|compile|install|targetinstall|tags)
> +		prepare|compile|install|targetinstall|tags)
> +			PTXDIST_OPTIMIZE_IO=true
> +			;&  # fallthrough
> +		extract)
>  			local cmd_post
>  			declare -a pkgs
>  
> @@ -1764,10 +1767,12 @@ EOF
>  			exit
>  			;;
>  		go|images)
> +			PTXDIST_OPTIMIZE_IO=true
>  			do_${cmd}
>  			exit
>  			;;
>  		image)
> +			PTXDIST_OPTIMIZE_IO=true
>  			if [ ${#} -eq 0 ]; then
>  				echo "No image given."
>  				exit 1
> @@ -1823,6 +1828,7 @@ EOF
>  			exit
>  			;;
>  		nfsroot)
> +			PTXDIST_OPTIMIZE_IO=true
>  			check_premake_compiler &&
>  			ptxd_make_log ptxd_make_nfsd
>  			exit
> @@ -2564,6 +2570,7 @@ setup_export() {
>  		PTXDIST_ICECC_CREATE_ENV \
>  		PTXDIST_LOG_PROMPT \
>  		PTXDIST_OUTPUT_SYNC \
> +		PTXDIST_OPTIMIZE_IO \
>  		PTXDIST_DIRTY \
>  		PTXDIST_FORCE \
>  		PTXDIST_PEDANTIC \
> diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
> index 8ee88827b1f8..8c09c531129e 100644
> --- a/scripts/lib/ptxd_lib_dgen.awk
> +++ b/scripts/lib/ptxd_lib_dgen.awk
> @@ -17,6 +17,7 @@ BEGIN {
>  	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
>  	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
>  	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
> +	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
>  	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
>  	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
>  	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
> @@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
>  	for (i = 1; i <= n; i++) {
>  		this_dep = PKG_to_pkg[this_DEP_array[i]]
>  
> -		if (PARALLEL != "-j1")
> +		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
> @@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
>  		else
>  			virtual = "base";
>  	}
> -	if (PARALLEL != "-j1")
> +	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de

Re: [ptxdist] [PATCH 1/2] ptxdist: add licensecheck command

[Michael Olbrich]

On Mon, May 25, 2020 at 11:28:19AM +0200, Roland Hieber wrote:
> Make it possible to check all MD5 sums of license files by building all
> report stages without also building the license report (which produces
> tons of LaTeX output).
> 
> Signed-off-by: Roland Hieber <rhi@pengutronix.de>
> ---
>  bin/ptxdist                              | 6 ++++++
>  doc/ref_parameter.inc                    | 3 +++
>  rules/post/ptxd_make_license_report.make | 2 ++
>  3 files changed, 11 insertions(+)
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 1e10c08d47f2..4a372ff6392c 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -752,6 +752,7 @@ Misc:
>  				it is known by "make"
>    printnext <var>		assumes that the contents of <var> is another
>  				variable and print the contents of this variable
> +  licensecheck			check md5sums of license files for all packages
>    list-packages			print a list of all selected packages
>    local-src <pkg> [<directory>]	overwrite a package source with a locally provided
>  				directory containing the sourcecode.
> @@ -1776,6 +1777,11 @@ EOF
>  			ptxd_make_log "${images[@]}"
>  			exit
>  			;;
> +		licensecheck)
> +			check_premake_compiler &&
> +			ptxd_make_log report
> +			exit
> +			;;
>  		list-packages)
>  			check_config &&
>  			check_deps || return
> diff --git a/doc/ref_parameter.inc b/doc/ref_parameter.inc
> index 74689b9d3cff..aaac8956f147 100644
> --- a/doc/ref_parameter.inc
> +++ b/doc/ref_parameter.inc
> @@ -234,6 +234,9 @@ Misc Actions
>    contents of this variable. Shell variables are currently not checked here.
>    All other rules for ``print`` apply.
>  
> +``licensecheck``
> +  For all selected packages, check the MD5 sums of license files.
> +
>  ``list-packages``
>    print a list of all selected packages. This list does not include the
>    host and cross tools.
> diff --git a/rules/post/ptxd_make_license_report.make b/rules/post/ptxd_make_license_report.make
> index 47dc176668d0..1a13c3ad4964 100644
> --- a/rules/post/ptxd_make_license_report.make
> +++ b/rules/post/ptxd_make_license_report.make
> @@ -15,6 +15,8 @@ PTX_PACKAGES_TARGET := \
>  	$(PTX_PACKAGES_INSTALL) \
>  	$(EXTRA_PACKAGES)
>  
> +report: $(addprefix $(STATEDIR)/,$(addsuffix .report,$(PTX_PACKAGES_SELECTED)))

Hmm, 'report' is a bit too generic for my taste. Just use 'licensecheck'
like the command. Also, You need to add it to PHONY.

mol


> +
>  PHONY += license-report
>  
>  license-report: \
> -- 
> 2.26.2
> 
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de
> 

-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de

[ptxdist] [PATCH v2 1/2] ptxdist: add licensecheck command

[Roland Hieber]

Make it possible to check all MD5 sums of license files by building all
report stages without also building the license report (which produces
tons of LaTeX output).

Signed-off-by: Roland Hieber <rhi@pengutronix.de>
---
v2: name the make target "licensecheck" instead of report, and add it to
    PHONY

 bin/ptxdist                              | 6 ++++++
 doc/ref_parameter.inc                    | 3 +++
 rules/post/ptxd_make_license_report.make | 4 ++++
 3 files changed, 13 insertions(+)

diff --git a/bin/ptxdist b/bin/ptxdist
index 1e10c08d47f2..6635a96a1cf5 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -752,6 +752,7 @@ Misc:
 				it is known by "make"
   printnext <var>		assumes that the contents of <var> is another
 				variable and print the contents of this variable
+  licensecheck			check md5sums of license files for all packages
   list-packages			print a list of all selected packages
   local-src <pkg> [<directory>]	overwrite a package source with a locally provided
 				directory containing the sourcecode.
@@ -1776,6 +1777,11 @@ EOF
 			ptxd_make_log "${images[@]}"
 			exit
 			;;
+		licensecheck)
+			check_premake_compiler &&
+			ptxd_make_log license-check
+			exit
+			;;
 		list-packages)
 			check_config &&
 			check_deps || return
diff --git a/doc/ref_parameter.inc b/doc/ref_parameter.inc
index 74689b9d3cff..aaac8956f147 100644
--- a/doc/ref_parameter.inc
+++ b/doc/ref_parameter.inc
@@ -234,6 +234,9 @@ Misc Actions
   contents of this variable. Shell variables are currently not checked here.
   All other rules for ``print`` apply.
 
+``licensecheck``
+  For all selected packages, check the MD5 sums of license files.
+
 ``list-packages``
   print a list of all selected packages. This list does not include the
   host and cross tools.
diff --git a/rules/post/ptxd_make_license_report.make b/rules/post/ptxd_make_license_report.make
index 47dc176668d0..a2d65cc2f450 100644
--- a/rules/post/ptxd_make_license_report.make
+++ b/rules/post/ptxd_make_license_report.make
@@ -15,6 +15,10 @@ PTX_PACKAGES_TARGET := \
 	$(PTX_PACKAGES_INSTALL) \
 	$(EXTRA_PACKAGES)
 
+PHONY += license-check
+
+license-check: $(addprefix $(STATEDIR)/,$(addsuffix .report,$(PTX_PACKAGES_SELECTED)))
+
 PHONY += license-report
 
 license-report: \
-- 
2.26.2


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de

Re: [ptxdist] [APPLIED] ptxdist: add licensecheck command

[Michael Olbrich]

Thanks, applied as d397238e966fe54cc2e9ad2234474e41ae863c1c.

Michael

[sent from post-receive hook]

On Fri, 05 Jun 2020 07:59:06 +0200, Roland Hieber <rhi@pengutronix.de> wrote:
> Make it possible to check all MD5 sums of license files by building all
> report stages without also building the license report (which produces
> tons of LaTeX output).
> 
> Signed-off-by: Roland Hieber <rhi@pengutronix.de>
> Message-Id: <20200529091339.8396-1-rhi@pengutronix.de>
> Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 1e10c08d47f2..6635a96a1cf5 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -752,6 +752,7 @@ Misc:
>  				it is known by "make"
>    printnext <var>		assumes that the contents of <var> is another
>  				variable and print the contents of this variable
> +  licensecheck			check md5sums of license files for all packages
>    list-packages			print a list of all selected packages
>    local-src <pkg> [<directory>]	overwrite a package source with a locally provided
>  				directory containing the sourcecode.
> @@ -1776,6 +1777,11 @@ EOF
>  			ptxd_make_log "${images[@]}"
>  			exit
>  			;;
> +		licensecheck)
> +			check_premake_compiler &&
> +			ptxd_make_log license-check
> +			exit
> +			;;
>  		list-packages)
>  			check_config &&
>  			check_deps || return
> diff --git a/doc/ref_parameter.inc b/doc/ref_parameter.inc
> index 74689b9d3cff..aaac8956f147 100644
> --- a/doc/ref_parameter.inc
> +++ b/doc/ref_parameter.inc
> @@ -234,6 +234,9 @@ Misc Actions
>    contents of this variable. Shell variables are currently not checked here.
>    All other rules for ``print`` apply.
>  
> +``licensecheck``
> +  For all selected packages, check the MD5 sums of license files.
> +
>  ``list-packages``
>    print a list of all selected packages. This list does not include the
>    host and cross tools.
> diff --git a/rules/post/ptxd_make_license_report.make b/rules/post/ptxd_make_license_report.make
> index 47dc176668d0..a2d65cc2f450 100644
> --- a/rules/post/ptxd_make_license_report.make
> +++ b/rules/post/ptxd_make_license_report.make
> @@ -15,6 +15,10 @@ PTX_PACKAGES_TARGET := \
>  	$(PTX_PACKAGES_INSTALL) \
>  	$(EXTRA_PACKAGES)
>  
> +PHONY += license-check
> +
> +license-check: $(addprefix $(STATEDIR)/,$(addsuffix .report,$(PTX_PACKAGES_SELECTED)))
> +
>  PHONY += license-report
>  
>  license-report: \

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de