Re: [ptxdist] [APPLIED] ptxd_lib_dgen: optimize ordering only when dependencies are needed

[Michael Olbrich <mol@pengutronix.de>]

Thanks, applied as 943ca0008b5add0d921153fb9ba365fcc33a8318.

Michael

[sent from post-receive hook]

On Fri, 05 Jun 2020 07:59:07 +0200, Roland Hieber <rhi@pengutronix.de> wrote:
> There are use cases where -j is useful to have on the command line to
> speed up runs with large amount of packages, but when the install stages
> are not needed at all. For example:
> 
>     ptxdist -j license-check
>     ptxdist -j make license-report
>     ptxdist -j extract <long list of package names>
> 
> Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency
> to extract") introduced better interleaving of extract stages to
> optimize IO load during full builds. This also had the effect that in
> the above cases all install stages of dependencies were built too, even
> when they are not needed at all.
> 
> Improve those use cases by only optimizing the extract stage ordering if
> at least the prepare stage(s) of the packages being requested are needed
> (since the prepare stage depends on the install stages of dependencies,
> those will be needed anyway).
> 
> Signed-off-by: Roland Hieber <rhi@pengutronix.de>
> Message-Id: <20200525202209.24858-1-rhi@pengutronix.de>
> Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 6635a96a1cf5..6b789d9a73fa 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -1579,7 +1579,10 @@ parse_second()
>  
>  ######## standard target, directly into make
>  
> -		extract|prepare|compile|install|targetinstall|tags)
> +		prepare|compile|install|targetinstall|tags)
> +			PTXDIST_OPTIMIZE_IO=true
> +			;&  # fallthrough
> +		extract)
>  			local cmd_post
>  			declare -a pkgs
>  
> @@ -1764,10 +1767,12 @@ EOF
>  			exit
>  			;;
>  		go|images)
> +			PTXDIST_OPTIMIZE_IO=true
>  			do_${cmd}
>  			exit
>  			;;
>  		image)
> +			PTXDIST_OPTIMIZE_IO=true
>  			if [ ${#} -eq 0 ]; then
>  				echo "No image given."
>  				exit 1
> @@ -1823,6 +1828,7 @@ EOF
>  			exit
>  			;;
>  		nfsroot)
> +			PTXDIST_OPTIMIZE_IO=true
>  			check_premake_compiler &&
>  			ptxd_make_log ptxd_make_nfsd
>  			exit
> @@ -2564,6 +2570,7 @@ setup_export() {
>  		PTXDIST_ICECC_CREATE_ENV \
>  		PTXDIST_LOG_PROMPT \
>  		PTXDIST_OUTPUT_SYNC \
> +		PTXDIST_OPTIMIZE_IO \
>  		PTXDIST_DIRTY \
>  		PTXDIST_FORCE \
>  		PTXDIST_PEDANTIC \
> diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk
> index 8ee88827b1f8..8c09c531129e 100644
> --- a/scripts/lib/ptxd_lib_dgen.awk
> +++ b/scripts/lib/ptxd_lib_dgen.awk
> @@ -17,6 +17,7 @@ BEGIN {
>  	DGEN_DEPS_PRE		= ENVIRON["PTX_DGEN_DEPS_PRE"];
>  	DGEN_DEPS_POST		= ENVIRON["PTX_DGEN_DEPS_POST"];
>  	DGEN_RULESFILES_MAKE	= ENVIRON["PTX_DGEN_RULESFILES_MAKE"];
> +	OPTIMIZE_IO		= ENVIRON["PTXDIST_OPTIMIZE_IO"]
>  	PTXDIST_TEMPDIR		= ENVIRON["PTXDIST_TEMPDIR"];
>  	PARALLEL		= ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"]
>  	DIRTY			= ENVIRON["PTXDIST_DIRTY"];
> @@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) {
>  	for (i = 1; i <= n; i++) {
>  		this_dep = PKG_to_pkg[this_DEP_array[i]]
>  
> -		if (PARALLEL != "-j1")
> +		if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  			print "$(STATEDIR)/" this_pkg	".extract:| " "$(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".extract.post:" DEP   " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
>  		print "$(STATEDIR)/" this_pkg	".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post"	> DGEN_DEPS_POST;
> @@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) {
>  		else
>  			virtual = "base";
>  	}
> -	if (PARALLEL != "-j1")
> +	if (PARALLEL != "-j1" && OPTIMIZE_IO == "true")
>  		print "$(STATEDIR)/" this_pkg ".extract:| "           "$(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".extract.post:" DEP           " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;
>  	print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP         " $(STATEDIR)/" virtual  ".install"	> DGEN_DEPS_POST;

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de