From mboxrd@z Thu Jan 1 00:00:00 1970 Return-path: Date: Mon, 25 May 2020 14:47:33 +0200 From: Michael Olbrich Message-ID: <20200525124733.GG16795@pengutronix.de> References: <20200525092820.818-1-rhi@pengutronix.de> <20200525092820.818-2-rhi@pengutronix.de> MIME-Version: 1.0 Content-Disposition: inline In-Reply-To: <20200525092820.818-2-rhi@pengutronix.de> Subject: Re: [ptxdist] [PATCH 2/2] ptxd_lib_dgen: optimize ordering only when packages are being fully built List-Id: PTXdist Development Mailing List List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: ptxdist@pengutronix.de Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: ptxdist-bounces@pengutronix.de Sender: "ptxdist" To: ptxdist@pengutronix.de Cc: Roland Hieber On Mon, May 25, 2020 at 11:28:20AM +0200, Roland Hieber wrote: > There are use cases where -j is useful to have on the command line to > speed up runs with large amount of packages, but when targetinstall > stages are not needed at all. For example: > > ptxdist -j license-check > ptxdist -j make license-report > ptxdist -j extract > > Commit 2e695987211c2c2e6e19 ("ptxd_lib_dgen: add order-only dependency > to extract") introduced better interleaving of extract stages to > optimise IO load during full builds. This also had the effect that in > the above cases all targetinstall stages of dependencies were built too, > even when they are not needed at all. Improve that use case by adding a > flag in the environment and only set it in case of 'ptxdist go', > 'ptxdist images' or 'ptxdist image '. > > Signed-off-by: Roland Hieber > --- > bin/ptxdist | 3 +++ > scripts/lib/ptxd_lib_dgen.awk | 5 +++-- > 2 files changed, 6 insertions(+), 2 deletions(-) > > diff --git a/bin/ptxdist b/bin/ptxdist > index 4a372ff6392c..6034d170fcdc 100755 > --- a/bin/ptxdist > +++ b/bin/ptxdist > @@ -1764,10 +1764,12 @@ EOF > exit > ;; > go|images) > + PTXDIST_OPTIMIZE_IO=true > do_${cmd} > exit > ;; > image) > + PTXDIST_OPTIMIZE_IO=true Hmmm, I think anything, that includes prepare stages should get this flag. Or maybe reverse the logic and flag commands where this should be skipped. I don't know which would be easier. Michael > if [ ${#} -eq 0 ]; then > echo "No image given." > exit 1 > @@ -2564,6 +2566,7 @@ setup_export() { > PTXDIST_ICECC_CREATE_ENV \ > PTXDIST_LOG_PROMPT \ > PTXDIST_OUTPUT_SYNC \ > + PTXDIST_OPTIMIZE_IO \ > PTXDIST_DIRTY \ > PTXDIST_FORCE \ > PTXDIST_PEDANTIC \ > diff --git a/scripts/lib/ptxd_lib_dgen.awk b/scripts/lib/ptxd_lib_dgen.awk > index 8ee88827b1f8..8c09c531129e 100644 > --- a/scripts/lib/ptxd_lib_dgen.awk > +++ b/scripts/lib/ptxd_lib_dgen.awk > @@ -17,6 +17,7 @@ BEGIN { > DGEN_DEPS_PRE = ENVIRON["PTX_DGEN_DEPS_PRE"]; > DGEN_DEPS_POST = ENVIRON["PTX_DGEN_DEPS_POST"]; > DGEN_RULESFILES_MAKE = ENVIRON["PTX_DGEN_RULESFILES_MAKE"]; > + OPTIMIZE_IO = ENVIRON["PTXDIST_OPTIMIZE_IO"] > PTXDIST_TEMPDIR = ENVIRON["PTXDIST_TEMPDIR"]; > PARALLEL = ENVIRON["PTXDIST_PARALLELMFLAGS_EXTERN"] > DIRTY = ENVIRON["PTXDIST_DIRTY"]; > @@ -503,7 +504,7 @@ function write_deps_pkg_active(this_PKG, this_pkg, prefix) { > for (i = 1; i <= n; i++) { > this_dep = PKG_to_pkg[this_DEP_array[i]] > > - if (PARALLEL != "-j1") > + if (PARALLEL != "-j1" && OPTIMIZE_IO == "true") > print "$(STATEDIR)/" this_pkg ".extract:| " "$(STATEDIR)/" this_dep ".install.post" > DGEN_DEPS_POST; > print "$(STATEDIR)/" this_pkg ".extract.post:" DEP " $(STATEDIR)/" this_dep ".install.post" > DGEN_DEPS_POST; > print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP " $(STATEDIR)/" this_dep ".install.post" > DGEN_DEPS_POST; > @@ -545,7 +546,7 @@ function write_deps_pkg_active_virtual(this_PKG, this_pkg, prefix) { > else > virtual = "base"; > } > - if (PARALLEL != "-j1") > + if (PARALLEL != "-j1" && OPTIMIZE_IO == "true") > print "$(STATEDIR)/" this_pkg ".extract:| " "$(STATEDIR)/" virtual ".install" > DGEN_DEPS_POST; > print "$(STATEDIR)/" this_pkg ".extract.post:" DEP " $(STATEDIR)/" virtual ".install" > DGEN_DEPS_POST; > print "$(STATEDIR)/" this_pkg ".install.unpack:" DEP " $(STATEDIR)/" virtual ".install" > DGEN_DEPS_POST; > -- > 2.26.2 > > > _______________________________________________ > ptxdist mailing list > ptxdist@pengutronix.de > -- Pengutronix e.K. | | Steuerwalder Str. 21 | http://www.pengutronix.de/ | 31137 Hildesheim, Germany | Phone: +49-5121-206917-0 | Amtsgericht Hildesheim, HRA 2686 | Fax: +49-5121-206917-5555 | _______________________________________________ ptxdist mailing list ptxdist@pengutronix.de