[ptxdist] [PATCH v2] image-root-cpio: Add custom compressor and compressor options.

[ptxdist] [PATCH v2] image-root-cpio: Add custom compressor and compressor options.

[Christian Melki]

* Add possibility for custom compression.
* Options are now issued with the compressor util.
* Fill in some more help regarding the methods.
* xz needs crc32 instead of crc64 for the kernel cpio.
* Set zstd compression level to the same as for the kernel (19).
* Use multithreading for zstd (deterministic, afaiu).

Actively decided against -T0 for xz, as compression
is not deterministic when number of threads varies.

Signed-off-by: Christian Melki <christian.melki@t2data.com>
---
 platforms/image-root-cpio.in | 48 +++++++++++++++++++++++++-----------
 rules/image-root-cpio.make   |  2 +-
 2 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/platforms/image-root-cpio.in b/platforms/image-root-cpio.in
index d3090dc29..fc95dd3f4 100644
--- a/platforms/image-root-cpio.in
+++ b/platforms/image-root-cpio.in
@@ -32,42 +32,62 @@ config IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
 	bool
 	prompt "gz"
 	help
-	  Compress root.cpio with gz
+	  Compress root.cpio with gz.
+	  Uses default compression level.
 
 config IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
 	bool
 	prompt "zstd"
 	help
-	  Compress root.cpio with zstd
+	  Compress root.cpio with zstd.
+	  Uses multithreaded, compression level 19.
 
 config IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
 	bool
 	prompt "xz"
 	help
-	  Compress root.cpio with xz
+	  Compress root.cpio with xz.
+	  Uses default compression level and
+	  crc32 checksum for kernel cpio compatibility.
 
 config IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
 	bool
 	prompt "lzop"
 	help
-	  Compress root.cpio with lzop
+	  Compress root.cpio with lzop.
+	  Uses default compression level.
+
+config IMAGE_ROOT_CPIO_COMPRESSION_MODE_CUSTOM
+	bool
+	prompt "custom"
+	help
+	  Specify your own compression.
+	  Default custom compression is set to gzip.
+	  You need to change compression suffix and
+	  compression utility with options.
+	  Beware that unknown compression utils are
+	  outside the ptxdist host-binary scope,
+	  so they must reside somewhere in your system
+	  for ptxdist to find.
 
 endchoice
 
 config IMAGE_ROOT_CPIO_COMPRESSION_SUFFIX
 	string
-	default ""		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
-	default ".gz"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
-	default ".zst"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
-	default ".xz"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
-	default ".lzo"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
+	default ""			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
+	default ".gz"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
+	default ".zst"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
+	default ".xz"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
+	default ".lzo"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
+	prompt  ".gz"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_CUSTOM
 
 config IMAGE_ROOT_CPIO_COMPRESSION_UTIL
 	string
-	default ""		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
-	default "gzip"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
-	default "zstd"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
-	default "xz"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
-	default "lzop"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
+	default ""			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
+	default "gzip"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
+	default "zstd -T0 -19"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
+	default "xz --check=crc32"	if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
+	default "lzop"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
+	prompt  "gzip"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_CUSTOM
 
 endif
diff --git a/rules/image-root-cpio.make b/rules/image-root-cpio.make
index fdb0ccfff..447a649c8 100644
--- a/rules/image-root-cpio.make
+++ b/rules/image-root-cpio.make
@@ -26,7 +26,7 @@ IMAGE_ROOT_CPIO_CONFIG	:= cpio.config
 
 IMAGE_ROOT_CPIO_ENV := \
 	FORMAT="newc" \
-	COMPRESS=$(call remove_quotes, $(PTXCONF_IMAGE_ROOT_CPIO_COMPRESSION_UTIL))
+	COMPRESS=$(PTXCONF_IMAGE_ROOT_CPIO_COMPRESSION_UTIL)
 
 ifdef PTXCONF_IMAGE_ROOT_CPIO
 $(IMAGE_ROOT_CPIO_IMAGE):
-- 
2.30.2


_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de

Re: [ptxdist] [PATCH v2] image-root-cpio: Add custom compressor and compressor options.

[Michael Olbrich]

Sorry, I missed this one.

It's not quite what I meant:

On Fri, Mar 04, 2022 at 10:55:49AM +0100, Christian Melki wrote:
> * Add possibility for custom compression.
> * Options are now issued with the compressor util.
> * Fill in some more help regarding the methods.
> * xz needs crc32 instead of crc64 for the kernel cpio.
> * Set zstd compression level to the same as for the kernel (19).
> * Use multithreading for zstd (deterministic, afaiu).
> 
> Actively decided against -T0 for xz, as compression
> is not deterministic when number of threads varies.
> 
> Signed-off-by: Christian Melki <christian.melki@t2data.com>
> ---
>  platforms/image-root-cpio.in | 48 +++++++++++++++++++++++++-----------
>  rules/image-root-cpio.make   |  2 +-
>  2 files changed, 35 insertions(+), 15 deletions(-)
> 
> diff --git a/platforms/image-root-cpio.in b/platforms/image-root-cpio.in
> index d3090dc29..fc95dd3f4 100644
> --- a/platforms/image-root-cpio.in
> +++ b/platforms/image-root-cpio.in
> @@ -32,42 +32,62 @@ config IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
>  	bool
>  	prompt "gz"
>  	help
> -	  Compress root.cpio with gz
> +	  Compress root.cpio with gz.
> +	  Uses default compression level.
>  
>  config IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
>  	bool
>  	prompt "zstd"
>  	help
> -	  Compress root.cpio with zstd
> +	  Compress root.cpio with zstd.
> +	  Uses multithreaded, compression level 19.
>  
>  config IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
>  	bool
>  	prompt "xz"
>  	help
> -	  Compress root.cpio with xz
> +	  Compress root.cpio with xz.
> +	  Uses default compression level and
> +	  crc32 checksum for kernel cpio compatibility.
>  
>  config IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
>  	bool
>  	prompt "lzop"
>  	help
> -	  Compress root.cpio with lzop
> +	  Compress root.cpio with lzop.
> +	  Uses default compression level.
> +
> +config IMAGE_ROOT_CPIO_COMPRESSION_MODE_CUSTOM
> +	bool
> +	prompt "custom"
> +	help
> +	  Specify your own compression.
> +	  Default custom compression is set to gzip.
> +	  You need to change compression suffix and
> +	  compression utility with options.
> +	  Beware that unknown compression utils are
> +	  outside the ptxdist host-binary scope,
> +	  so they must reside somewhere in your system
> +	  for ptxdist to find.

not here...

>  
>  endchoice

but here:

config IMAGE_ROOT_CPIO_CUSTOM_COMPRESSION
	bool
	prompt "customize compression command"
	help
	  This allows specifying additional arguments to the compressor
	  command. Note that the command itself should not be changed.
	  Otherwise ptxdist cannot ensure that the tool is available.

>  
>  config IMAGE_ROOT_CPIO_COMPRESSION_SUFFIX
>  	string
> -	default ""		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
> -	default ".gz"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
> -	default ".zst"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
> -	default ".xz"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
> -	default ".lzo"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
> +	default ""			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
> +	default ".gz"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
> +	default ".zst"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
> +	default ".xz"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
> +	default ".lzo"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
> +	prompt  ".gz"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_CUSTOM

no changes to the suffix


>  config IMAGE_ROOT_CPIO_COMPRESSION_UTIL
>  	string

	prompt "compression tool" if IMAGE_ROOT_CPIO_CUSTOM_COMPRESSION

> -	default ""		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
> -	default "gzip"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
> -	default "zstd"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
> -	default "xz"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
> -	default "lzop"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP
> +	default ""			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_NONE
> +	default "gzip"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_GZ
> +	default "zstd -T0 -19"		if IMAGE_ROOT_CPIO_COMPRESSION_MODE_ZSTD
> +	default "xz --check=crc32"	if IMAGE_ROOT_CPIO_COMPRESSION_MODE_XZ
> +	default "lzop"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_LZOP

These are ok.

So when IMAGE_ROOT_CPIO_CUSTOM_COMPRESSION is enabled then
IMAGE_ROOT_CPIO_COMPRESSION_UTIL will show up in the menu and the command
can be modified. The dependencies will still be ok unless the user changes
the tool itself.

Michael

> +	prompt  "gzip"			if IMAGE_ROOT_CPIO_COMPRESSION_MODE_CUSTOM

>  
>  endif
> diff --git a/rules/image-root-cpio.make b/rules/image-root-cpio.make
> index fdb0ccfff..447a649c8 100644
> --- a/rules/image-root-cpio.make
> +++ b/rules/image-root-cpio.make
> @@ -26,7 +26,7 @@ IMAGE_ROOT_CPIO_CONFIG	:= cpio.config
>  
>  IMAGE_ROOT_CPIO_ENV := \
>  	FORMAT="newc" \
> -	COMPRESS=$(call remove_quotes, $(PTXCONF_IMAGE_ROOT_CPIO_COMPRESSION_UTIL))
> +	COMPRESS=$(PTXCONF_IMAGE_ROOT_CPIO_COMPRESSION_UTIL)
>  
>  ifdef PTXCONF_IMAGE_ROOT_CPIO
>  $(IMAGE_ROOT_CPIO_IMAGE):
> -- 
> 2.30.2
> 
> 
> _______________________________________________
> ptxdist mailing list
> ptxdist@pengutronix.de
> To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de
> 

-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

_______________________________________________
ptxdist mailing list
ptxdist@pengutronix.de
To unsubscribe, send a mail with subject "unsubscribe" to ptxdist-request@pengutronix.de