[ptxdist] [PATCH v2 2/2] ptxd_lib_dgen: optimize ordering only when dependencies are needed

[Roland Hieber <rhi@pengutronix.de>]

There are use cases where -j is useful to have on the command line to
speed up runs with large amount of packages, but when the install stages
are not needed at all. For example:

    ptxdist -j license-check
    ptxdist -j make license-report
    ptxdist -j extract <long list of package names>

Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
to extract") introduced better interleaving of extract stages to
optimize IO load during full builds. This also had the effect that in
the above cases all install stages of dependencies were built too, even
when they are not needed at all.

Improve those use cases by only optimizing the extract stage ordering if
at least the prepare stage(s) of the packages being requested are needed
(since the prepare stage depends on the install stages of dependencies,
those will be needed anyway).

Signed-off-by: Roland Hieber <rhi@pengutronix.de>
---
v2: also set PTXDIST_OPTIMIZE_IO in prepare, compile, install,
    targetinstall, tags, nfsroot; and adapt log message accordingly

On Mon, May 25, 2020 at 02:47:33PM +0200, Michael Olbrich wrote:
> Hmmm, I think anything, that includes prepare stages should get this flag.
> Or maybe reverse the logic and flag commands where this should be skipped.
> I don't know which would be easier.
> 
> Michael

Like this?
A positive list is still easier :)

 - Roland

---
 bin/ptxdist                   | 9 ++++++++-
 scripts/lib/ptxd_lib_dgen.awk | 5 +++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/bin/ptxdist b/bin/ptxdist
index 4a372ff6392c..0d85c1290868 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -1579,7 +1579,10 @@ parse_second()
 
 ######## standard target, directly into make
 
-		extract|prepare|compile|install|targetinstall|tags)
+		prepare|compile|install|targetinstall|tags)
+			PTXDIST_OPTIMIZE_IO=true
+			;&  # fallthrough
+		extract)
 			local cmd_post
 			declare -a pkgs
 
@@ -1764,10 +1767,12 @@ EOF
 			exit
 			;;
 		go|images)
+			PTXDIST_OPTIMIZE_IO=true
 			do_${cmd}
 			exit
 			;;
 		image)
+			PTXDIST_OPTIMIZE_IO=true
 			if [ ${#} -eq 0 ]; then
 				echo "No image given."
 				exit 1
@@ -1823,6 +1828,7 @@ EOF
 			exit
 			;;
 		nfsroot)
+			PTXDIST_OPTIMIZE_IO=true
 			check_premake_compiler &&
 			ptxd_make_log ptxd_make_nfsd
 			exit
@@ -2564,6 +2570,7 @@ setup_export() {
 		PTXDIST_ICECC_CREATE_ENV \
 		PTXDIST_LOG_PROMPT \
 		PTXDIST_OUTPUT_SYNC \
+		PTXDIST_OPTIMIZE_IO \
 		PTXDIST_DIRTY \
 		PTXDIST_FORCE \
 		PTXDIST_PEDANTIC \
diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
index 8ee88827b1f8..8c09c531129e 100644
--- a/scripts/lib/ptxd_lib_dgen.awk
+++ b/scripts/lib/ptxd_lib_dgen.awk
@@ -17,6 +17,7 @@ BEGIN {
 	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
 	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
 	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
+	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
 	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
 	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
 	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
@@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
 	for (i = 1; i <= n; i++) {
 		this_dep = PKG_to_pkg[this_DEP_array[i]]
 
-		if (PARALLEL != "-j1")
+		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
 			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
 		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
 		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
@@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
 		else
 			virtual = "base";
 	}
-	if (PARALLEL != "-j1")
+	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
 		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
 	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
-- 
2.26.2


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de