[ptxdist] [PATCH 1/3] RFC: ptxd_make_world: Extract CPE for packages

[ptxdist] [PATCH 1/3] RFC: ptxd_make_world: Extract CPE for packages

[Simon Falsig]

If a package specifies a CPE or CPE_VENDOR and CPE_PRODUCT, this is
extracted into the fast report for that package. If no CPE is
specified, or not both of CPE_VENDOR and CPE_PRODUCT, then no value is
added.

By default, the existing VERSION is used, but can be overridden with
CPE_VERSION.

Constructed CPEs are validated against the official CPE regex.

The CPE (Common Platform Enumerator) allows matching CVEs to specific
packages, and see if these apply to a specific deployment.
---
 rules/post/ptxd_make_world_common.make | 4 ++++
 scripts/lib/ptxd_make_world_report.sh  | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/rules/post/ptxd_make_world_common.make b/rules/post/ptxd_make_world_common.make
index 4b6f691b6..189bc4ec9 100644
--- a/rules/post/ptxd_make_world_common.make
+++ b/rules/post/ptxd_make_world_common.make
@@ -80,6 +80,10 @@ world/env/impl = \
 	pkg_PKG="$(call ptx/escape,$(1))"					\
 	pkg_pkg="$(call ptx/escape,$($(1)))"					\
 	pkg_version="$(call ptx/escape,$($(1)_VERSION))"			\
+	pkg_cpe_vendor="$(call ptx/escape,$($(1)_CPE_VENDOR))"			\
+	pkg_cpe_product="$(call ptx/escape,$($(1)_CPE_PRODUCT))"		\
+	pkg_cpe_version="$(call ptx/escape,$($(1)_CPE_VERSION))"		\
+	pkg_cpe="$(call ptx/escape,$($(1)_CPE))"				\
 	pkg_config="$(call ptx/escape,$($(1)_CONFIG))"				\
 	pkg_ref_config="$(call ptx/escape,$($(1)_REF_CONFIG))"			\
 	pkg_cargo_lock="$(call ptx/escape,$($(1)_CARGO_LOCK))"			\
diff --git a/scripts/lib/ptxd_make_world_report.sh b/scripts/lib/ptxd_make_world_report.sh
index 2c02e81f7..37fa2b89e 100644
--- a/scripts/lib/ptxd_make_world_report.sh
+++ b/scripts/lib/ptxd_make_world_report.sh
@@ -72,6 +72,15 @@ ptxd_make_world_report_yaml() {
     do_list "rundeps:" "${pkg_run_deps}"
     do_echo "config:" "${pkg_config}"
     do_echo "version:" "${pkg_version}"
+    if [ ! -n "${pkg_cpe_version}" -a ! -n "${pkg_cpe}" ]; then
+	# Default to using pkg_version for the CPE string, unless _CPE_VERSION or _CPE are explicitly
+	# specified. In the case of the latter, there's no need to keep track of the version separately.
+	pkg_cpe_version="${pkg_version}"
+    fi
+    do_echo "cpe:" "${pkg_cpe}"
+    do_echo "cpe_vendor:" "${pkg_cpe_vendor}"
+    do_echo "cpe_product:" "${pkg_cpe_product}"
+    do_echo "cpe_version:" "${pkg_cpe_version}"
     do_list "url:" "${pkg_url}"
     do_echo "md5:" "${pkg_md5}"
     do_echo "source:" "${pkg_src}"
-- 
2.25.1

[ptxdist] [PATCH 2/3] RFC: Add CPE for a few packages

[Simon Falsig]

Just to see how this could look for a handful of packages. Note that all
of these have a different way of specifying the vendor ID (one is
$PACKAGE_project, one is just $PACKAGE, one is something completely
different).
---
 rules/acl.make            | 1 +
 rules/busybox.make        | 2 ++
 rules/wpa_supplicant.make | 3 +++
 3 files changed, 6 insertions(+)

diff --git a/rules/acl.make b/rules/acl.make
index 265a8790c..8f1724fb1 100644
--- a/rules/acl.make
+++ b/rules/acl.make
@@ -21,6 +21,7 @@ ACL_SUFFIX	:= tar.gz
 ACL_URL		:= http://download.savannah.gnu.org/releases/acl/$(ACL).$(ACL_SUFFIX)
 ACL_SOURCE	:= $(SRCDIR)/$(ACL).$(ACL_SUFFIX)
 ACL_DIR		:= $(BUILDDIR)/$(ACL)
+ACL_CPE		:= cpe:2.3:a:acl_project:acl:$(ACL_VERSION):*:*:*:*:*:*:*
 ACL_LICENSE	:= LGPL-2.1-or-later
 ACL_LICENSE_FILES:= file://doc/COPYING.LGPL;md5=9e9a206917f8af112da634ce3ab41764
 ifdef PTXCONF_ACL_TOOLS
diff --git a/rules/busybox.make b/rules/busybox.make
index 929f721c7..fb0738730 100644
--- a/rules/busybox.make
+++ b/rules/busybox.make
@@ -22,6 +22,8 @@ BUSYBOX_URL	:= https://www.busybox.net/downloads/$(BUSYBOX).$(BUSYBOX_SUFFIX)
 BUSYBOX_SOURCE	:= $(SRCDIR)/$(BUSYBOX).$(BUSYBOX_SUFFIX)
 BUSYBOX_DIR	:= $(BUILDDIR)/$(BUSYBOX)
 BUSYBOX_KCONFIG	:= $(BUSYBOX_DIR)/Config.in
+BUSYBOX_CPE_VENDOR	:= busybox
+BUSYBOX_CPE_PRODUCT	:= busybox
 BUSYBOX_LICENSE	:= GPL-2.0-only
 BUSYBOX_LICENSE_FILES := file://LICENSE;md5=de10de48642ab74318e893a61105afbb
 
diff --git a/rules/wpa_supplicant.make b/rules/wpa_supplicant.make
index 644df4fcb..c2172853d 100644
--- a/rules/wpa_supplicant.make
+++ b/rules/wpa_supplicant.make
@@ -26,6 +26,9 @@ WPA_SUPPLICANT_SUBDIR	:= $(WPA_SUPPLICANT_NAME)
 # Use '=' to delay $(shell ...) calls until this is needed
 WPA_SUPPLICANT_CONFIG	 = $(call ptx/get-alternative, config/wpasupplicant, defconfig)
 WPA_SUPPLICANT_DOTCONFIG:= $(BUILDDIR)/$(WPA_SUPPLICANT)/$(WPA_SUPPLICANT_SUBDIR)/.config
+WPA_SUPPLICANT_CPE_VENDOR := w1.fi
+WPA_SUPPLICANT_CPE_PRODUCT := wpa_supplicant
+WPA_SUPPLICANT_CPE_VERSION := $(WPA_SUPPLICANT_VERSION)
 WPA_SUPPLICANT_LICENSE	:= GPL-2.0-only
 
 # ----------------------------------------------------------------------------
-- 
2.25.1

[ptxdist] [PATCH 3/3] RFC: sbom_report: Add support

[Simon Falsig]

This provides support for building SBOMs in CycloneDX format.

A target is added alongside the other reports, that (based on the
fast-bsp-report) extracts name, version, cpe and license of each target
package, and puts these into a final sbom-report in CycloneDX/JSON
format.

This requires a working Python3 setup with the cyclonedx-python-lib
package installed.
---
 bin/ptxdist                          |  3 +-
 rules/host-system-python3.in         |  3 ++
 rules/host-system-python3.make       |  6 ++++
 rules/post/ptxd_make_report.make     | 16 +++++++--
 rules/sbom-report.in                 |  9 +++++
 scripts/lib/ptxd_make_report.sh      | 16 +++++++++
 scripts/lib/ptxd_make_sbom_report.py | 52 ++++++++++++++++++++++++++++
 7 files changed, 102 insertions(+), 3 deletions(-)
 create mode 100644 rules/sbom-report.in
 create mode 100644 scripts/lib/ptxd_make_sbom_report.py

diff --git a/bin/ptxdist b/bin/ptxdist
index 77cad673b..9e62d2bc0 100755
--- a/bin/ptxdist
+++ b/bin/ptxdist
@@ -792,6 +792,7 @@ Misc:
   full-bsp-report		generate a yaml file that describes the BSP and
 				all packages. More data but will build all
 				packages if necessary.
+  sbom-report			generate a CycloneDX json SBOM
   print <var>			print the contents of a variable, in the way
 				it is known by "make"
   printnext <var>		assumes that the contents of <var> is another
@@ -1841,7 +1842,7 @@ EOF
 			ptxd_make_log export_src EXPORTDIR="${1}"
 			exit
 			;;
-		fast-bsp-report|full-bsp-report)
+		fast-bsp-report|full-bsp-report|sbom-report)
 			check_premake_compiler &&
 			ptxd_make_log "${cmd}"
 			exit
diff --git a/rules/host-system-python3.in b/rules/host-system-python3.in
index 25201e93f..c158a295f 100644
--- a/rules/host-system-python3.in
+++ b/rules/host-system-python3.in
@@ -35,4 +35,7 @@ config HOST_SYSTEM_PYTHON3_PYELFTOOLS
 config HOST_SYSTEM_PYTHON3_PYYAML
 	bool
 
+config HOST_SYSTEM_PYTHON3_CYCLONEDX
+	bool
+
 endif
diff --git a/rules/host-system-python3.make b/rules/host-system-python3.make
index 3688cf09d..f4979a453 100644
--- a/rules/host-system-python3.make
+++ b/rules/host-system-python3.make
@@ -88,6 +88,12 @@ ifdef PTXCONF_HOST_SYSTEM_PYTHON3_PYYAML
 		ptxd_bailout "Python pyyaml module not found! \
 	Please install python3-yaml (debian)";
 endif
+ifdef PTXCONF_HOST_SYSTEM_PYTHON3_CYCLONEDX
+	@echo "Checking for Python cyclonedx-python-lib ..."
+	@$(SYSTEMPYTHON3) -c 'import cyclonedx.factory' 2>/dev/null || \
+		ptxd_bailout "Python cyclonedx-python-lib module not found! \
+	Please install cyclonedx-python-lib (pip)";
+endif
 
 	@$(call touch)
 
diff --git a/rules/post/ptxd_make_report.make b/rules/post/ptxd_make_report.make
index eecd2a577..529b1c78e 100644
--- a/rules/post/ptxd_make_report.make
+++ b/rules/post/ptxd_make_report.make
@@ -10,7 +10,10 @@ ptx/report-env = \
 	$(image/env) \
 	ptx_report_target="$(strip $(1))" \
 	ptx_packages_selected="$(filter-out $(IMAGE_PACKAGES),$(PTX_PACKAGES_SELECTED))" \
-	ptx_image_packages="$(IMAGE_PACKAGES)"
+	ptx_image_packages="$(IMAGE_PACKAGES)" \
+	ptx_target_packages="$(PACKAGES)" \
+	ptx_system_python3="$(SYSTEMPYTHON3)"
+
 
 PHONY += full-bsp-report
 full-bsp-report: $(RELEASEDIR)/full-bsp-report.yaml
@@ -26,13 +29,22 @@ $(RELEASEDIR)/full-bsp-report.yaml: \
 	@$(call ptx/report-env, $@) ptxd_make_full_bsp_report
 	@$(call finish)
 
+
 PHONY += fast-bsp-report
 fast-bsp-report: $(RELEASEDIR)/fast-bsp-report.yaml
 
-
 $(RELEASEDIR)/fast-bsp-report.yaml: $(addprefix $(STATEDIR)/,$(addsuffix .fast-report,$(PTX_PACKAGES_SELECTED)))
 	@$(call targetinfo)
 	@$(call ptx/report-env, $@) ptxd_make_fast_bsp_report
 	@$(call finish)
 
+
+PHONY += sbom-report
+sbom-report: $(RELEASEDIR)/sbom-report.json
+
+$(RELEASEDIR)/sbom-report.json: $(STATEDIR)/host-system-python3.install $(addprefix $(STATEDIR)/,$(addsuffix .fast-report,$(PACKAGES)))
+	@$(call targetinfo)
+	@$(call ptx/report-env, $@) ptxd_make_sbom_report
+	@$(call finish)
+
 # vim: syntax=make
diff --git a/rules/sbom-report.in b/rules/sbom-report.in
new file mode 100644
index 000000000..311ad5a23
--- /dev/null
+++ b/rules/sbom-report.in
@@ -0,0 +1,9 @@
+## SECTION=ptxdist_options
+
+config ENABLE_SBOM_REPORT
+	bool
+	select HOST_SYSTEM_PYTHON3
+	select HOST_SYSTEM_PYTHON3_CYCLONEDX
+	prompt "enable sbom report generation"
+	help
+	  This enables the generation of an SBOM report, through ptxdist sbom-report.
diff --git a/scripts/lib/ptxd_make_report.sh b/scripts/lib/ptxd_make_report.sh
index 885f07828..f2e92e6be 100644
--- a/scripts/lib/ptxd_make_report.sh
+++ b/scripts/lib/ptxd_make_report.sh
@@ -146,3 +146,19 @@ ptxd_make_fast_bsp_report() {
 }
 export -f ptxd_make_fast_bsp_report
 
+ptxd_make_sbom_report() {
+    local -a ptxd_reply
+    local pkg_lic pkg
+
+    ptxd_make_layer_init || return
+
+    echo "Generating $(ptxd_print_path "${ptx_report_target}") ..."
+    echo
+    
+    mkdir -p "$(dirname "${ptx_report_target}")" &&
+    "${ptx_system_python3}" ${PTXDIST_LIB_DIR}/ptxd_make_sbom_report.py "${ptx_report_dir}/fast/" ${ptx_target_packages} > ${PTXDIST_TEMPDIR}/sbom-report &&
+    mv "${PTXDIST_TEMPDIR}/sbom-report" "${ptx_report_target}" ||
+    ptxd_bailout "failed to create SBOM report"
+}
+export -f ptxd_make_sbom_report
+
diff --git a/scripts/lib/ptxd_make_sbom_report.py b/scripts/lib/ptxd_make_sbom_report.py
new file mode 100644
index 000000000..a9ec36697
--- /dev/null
+++ b/scripts/lib/ptxd_make_sbom_report.py
@@ -0,0 +1,52 @@
+from cyclonedx.factory.license import LicenseFactory
+from cyclonedx.model.bom import Bom
+from cyclonedx.model.component import Component
+from cyclonedx.output.json import JsonV1Dot4
+import sys
+import re
+
+lFac = LicenseFactory()
+bom = Bom()
+
+for i in range(2, len(sys.argv)):
+    pkg_report = sys.argv[1] + sys.argv[i] + ".yaml"
+    with open(pkg_report, 'r') as file:
+        content = file.read()
+        name_ = re.search("name: \'(.+)\'", content).group(1)
+        version_ = re.search("version: \'(.+)\'", content).group(1)
+
+        # First see if we have a full CPE specified, then use that
+        cpe_match = re.search("cpe: \'(.+)\'", content)
+        cpe_ = None
+        if cpe_match is not None:
+            cpe_ = cpe_match.group(1)
+        else:
+            # See if we have the individual components
+            cpe_vendor_match = re.search("cpe_vendor: \'(.+)\'", content)
+            cpe_product_match = re.search("cpe_product: \'(.+)\'", content)
+            cpe_version_match = re.search("cpe_version: \'(.+)\'", content)
+            if cpe_vendor_match is not None and cpe_product_match is not None and cpe_version_match is not None:
+                cpe_ = "cpe:2.3:a:{vendor}:{product}:{version}:*:*:*:*:*:*:*".format(vendor=cpe_vendor_match.group(1),
+                                                                                     product=cpe_product_match.group(1),
+                                                                                     version=cpe_version_match.group(1))
+
+        if cpe_ is not None:
+            # We have a CPE, let's validate it. Regex from: https://csrc.nist.gov/schema/cpe/2.3/cpe-naming_2.3.xsd
+            if not re.fullmatch("cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!\"#$$%&'\(\)\+,/:;"
+                                "<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)"
+                                "|[\*\-]))(:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!\"#$$%&'\(\)\+,/:;<=>@\[\]\^`\{\|}~]"
+                                "))+(\?*|\*?))|[\*\-])){4}", cpe_):
+                raise ValueError("Constructed CPE is not valid: {cpe}".format(cpe=cpe_))
+
+        licenses_ = re.search("licenses: \'(.+)\'", content).group(1)
+        comp = Component(
+            name=name_,
+            version=version_,
+            cpe=cpe_,
+            licenses=[lFac.make_from_string(licenses_)],
+            bom_ref=name_ + "@" + version_
+        )
+        bom.components.add(comp)
+
+serializedJSON = JsonV1Dot4(bom).output_as_string()
+print(serializedJSON)
-- 
2.25.1

Re: [ptxdist] [PATCH 1/3] RFC: ptxd_make_world: Extract CPE for packages

[Michael Olbrich]

On Mon, Feb 19, 2024 at 05:56:15PM +0100, Simon Falsig wrote:
> If a package specifies a CPE or CPE_VENDOR and CPE_PRODUCT, this is
> extracted into the fast report for that package. If no CPE is
> specified, or not both of CPE_VENDOR and CPE_PRODUCT, then no value is
> added.
> 
> By default, the existing VERSION is used, but can be overridden with
> CPE_VERSION.
> 
> Constructed CPEs are validated against the official CPE regex.
> 
> The CPE (Common Platform Enumerator) allows matching CVEs to specific
> packages, and see if these apply to a specific deployment.

So we need to change the plan on how to do this a bit. I've looked into
this stuff some more. Specifically what yocto is doing here. This is a bit
of a mess. Take a look at this[1]:

[...]
CVE_PRODUCT = "haxx:curl haxx:libcurl curl:curl curl:libcurl libcurl:libcurl daniel_stenberg:curl"
[...]

There are packages that need multiple vendor/product combinations :-/. So
the product is a list. And if an entry contains a ':' then it's
<vendor>:<product>. Otherwise '*' is used for the vendor.

And please use _CVE_PRODUCT (instead of _CPE_PRODUCT) so it's called the
same.

[1] https://git.openembedded.org/openembedded-core/tree/meta/recipes-support/curl/curl_8.6.0.bb?id=efebd6a8824769137a21674e2bfe1c059a41758a#n20

> ---
>  rules/post/ptxd_make_world_common.make | 4 ++++
>  scripts/lib/ptxd_make_world_report.sh  | 9 +++++++++
>  2 files changed, 13 insertions(+)
> 
> diff --git a/rules/post/ptxd_make_world_common.make b/rules/post/ptxd_make_world_common.make
> index 4b6f691b6..189bc4ec9 100644
> --- a/rules/post/ptxd_make_world_common.make
> +++ b/rules/post/ptxd_make_world_common.make
> @@ -80,6 +80,10 @@ world/env/impl = \
>  	pkg_PKG="$(call ptx/escape,$(1))"					\
>  	pkg_pkg="$(call ptx/escape,$($(1)))"					\
>  	pkg_version="$(call ptx/escape,$($(1)_VERSION))"			\
> +	pkg_cpe_vendor="$(call ptx/escape,$($(1)_CPE_VENDOR))"			\
> +	pkg_cpe_product="$(call ptx/escape,$($(1)_CPE_PRODUCT))"		\
> +	pkg_cpe_version="$(call ptx/escape,$($(1)_CPE_VERSION))"		\
> +	pkg_cpe="$(call ptx/escape,$($(1)_CPE))"				\
>  	pkg_config="$(call ptx/escape,$($(1)_CONFIG))"				\
>  	pkg_ref_config="$(call ptx/escape,$($(1)_REF_CONFIG))"			\
>  	pkg_cargo_lock="$(call ptx/escape,$($(1)_CARGO_LOCK))"			\
> diff --git a/scripts/lib/ptxd_make_world_report.sh b/scripts/lib/ptxd_make_world_report.sh
> index 2c02e81f7..37fa2b89e 100644
> --- a/scripts/lib/ptxd_make_world_report.sh
> +++ b/scripts/lib/ptxd_make_world_report.sh
> @@ -72,6 +72,15 @@ ptxd_make_world_report_yaml() {
>      do_list "rundeps:" "${pkg_run_deps}"
>      do_echo "config:" "${pkg_config}"
>      do_echo "version:" "${pkg_version}"
> +    if [ ! -n "${pkg_cpe_version}" -a ! -n "${pkg_cpe}" ]; then
> +	# Default to using pkg_version for the CPE string, unless _CPE_VERSION or _CPE are explicitly
> +	# specified. In the case of the latter, there's no need to keep track of the version separately.
> +	pkg_cpe_version="${pkg_version}"
> +    fi

don't do this here. This fallback should be handled in the script.

> +    do_echo "cpe:" "${pkg_cpe}"

This should be a list in case we need more than one.

> +    do_echo "cpe_vendor:" "${pkg_cpe_vendor}"

Same here as described above.

Michael

> +    do_echo "cpe_product:" "${pkg_cpe_product}"
> +    do_echo "cpe_version:" "${pkg_cpe_version}"
>      do_list "url:" "${pkg_url}"
>      do_echo "md5:" "${pkg_md5}"
>      do_echo "source:" "${pkg_src}"
> -- 
> 2.25.1
> 
> 
> 

-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

Re: [ptxdist] [PATCH 3/3] RFC: sbom_report: Add support

[Michael Olbrich]

On Mon, Feb 19, 2024 at 05:56:17PM +0100, Simon Falsig wrote:
> This provides support for building SBOMs in CycloneDX format.
> 
> A target is added alongside the other reports, that (based on the
> fast-bsp-report) extracts name, version, cpe and license of each target
> package, and puts these into a final sbom-report in CycloneDX/JSON
> format.
> 
> This requires a working Python3 setup with the cyclonedx-python-lib
> package installed.
> ---
>  bin/ptxdist                          |  3 +-
>  rules/host-system-python3.in         |  3 ++
>  rules/host-system-python3.make       |  6 ++++
>  rules/post/ptxd_make_report.make     | 16 +++++++--
>  rules/sbom-report.in                 |  9 +++++
>  scripts/lib/ptxd_make_report.sh      | 16 +++++++++
>  scripts/lib/ptxd_make_sbom_report.py | 52 ++++++++++++++++++++++++++++
>  7 files changed, 102 insertions(+), 3 deletions(-)
>  create mode 100644 rules/sbom-report.in
>  create mode 100644 scripts/lib/ptxd_make_sbom_report.py
> 
> diff --git a/bin/ptxdist b/bin/ptxdist
> index 77cad673b..9e62d2bc0 100755
> --- a/bin/ptxdist
> +++ b/bin/ptxdist
> @@ -792,6 +792,7 @@ Misc:
>    full-bsp-report		generate a yaml file that describes the BSP and
>  				all packages. More data but will build all
>  				packages if necessary.
> +  sbom-report			generate a CycloneDX json SBOM

So I'm not so sure if I want a separate command for this. I think it would
be better to build this with the images.

>    print <var>			print the contents of a variable, in the way
>  				it is known by "make"
>    printnext <var>		assumes that the contents of <var> is another
> @@ -1841,7 +1842,7 @@ EOF
>  			ptxd_make_log export_src EXPORTDIR="${1}"
>  			exit
>  			;;
> -		fast-bsp-report|full-bsp-report)
> +		fast-bsp-report|full-bsp-report|sbom-report)
>  			check_premake_compiler &&
>  			ptxd_make_log "${cmd}"
>  			exit
> diff --git a/rules/host-system-python3.in b/rules/host-system-python3.in
> index 25201e93f..c158a295f 100644
> --- a/rules/host-system-python3.in
> +++ b/rules/host-system-python3.in
> @@ -35,4 +35,7 @@ config HOST_SYSTEM_PYTHON3_PYELFTOOLS
>  config HOST_SYSTEM_PYTHON3_PYYAML
>  	bool
>  
> +config HOST_SYSTEM_PYTHON3_CYCLONEDX
> +	bool
> +
>  endif
> diff --git a/rules/host-system-python3.make b/rules/host-system-python3.make
> index 3688cf09d..f4979a453 100644
> --- a/rules/host-system-python3.make
> +++ b/rules/host-system-python3.make
> @@ -88,6 +88,12 @@ ifdef PTXCONF_HOST_SYSTEM_PYTHON3_PYYAML
>  		ptxd_bailout "Python pyyaml module not found! \
>  	Please install python3-yaml (debian)";
>  endif
> +ifdef PTXCONF_HOST_SYSTEM_PYTHON3_CYCLONEDX
> +	@echo "Checking for Python cyclonedx-python-lib ..."
> +	@$(SYSTEMPYTHON3) -c 'import cyclonedx.factory' 2>/dev/null || \
> +		ptxd_bailout "Python cyclonedx-python-lib module not found! \
> +	Please install cyclonedx-python-lib (pip)";
> +endif
>  
>  	@$(call touch)
>  

This part (or whatever replaces this) should be a separate patch.

> diff --git a/rules/post/ptxd_make_report.make b/rules/post/ptxd_make_report.make
> index eecd2a577..529b1c78e 100644
> --- a/rules/post/ptxd_make_report.make
> +++ b/rules/post/ptxd_make_report.make
> @@ -10,7 +10,10 @@ ptx/report-env = \
>  	$(image/env) \
>  	ptx_report_target="$(strip $(1))" \
>  	ptx_packages_selected="$(filter-out $(IMAGE_PACKAGES),$(PTX_PACKAGES_SELECTED))" \
> -	ptx_image_packages="$(IMAGE_PACKAGES)"
> +	ptx_image_packages="$(IMAGE_PACKAGES)" \
> +	ptx_target_packages="$(PACKAGES)" \

This should not be needed. See below.

> +	ptx_system_python3="$(SYSTEMPYTHON3)"
> +

Just the normal she-bang should to the right thing.

>  
>  PHONY += full-bsp-report
>  full-bsp-report: $(RELEASEDIR)/full-bsp-report.yaml
> @@ -26,13 +29,22 @@ $(RELEASEDIR)/full-bsp-report.yaml: \
>  	@$(call ptx/report-env, $@) ptxd_make_full_bsp_report
>  	@$(call finish)
>  
> +
>  PHONY += fast-bsp-report
>  fast-bsp-report: $(RELEASEDIR)/fast-bsp-report.yaml
>  
> -

Unrelated changes.

>  $(RELEASEDIR)/fast-bsp-report.yaml: $(addprefix $(STATEDIR)/,$(addsuffix .fast-report,$(PTX_PACKAGES_SELECTED)))
>  	@$(call targetinfo)
>  	@$(call ptx/report-env, $@) ptxd_make_fast_bsp_report
>  	@$(call finish)
>  
> +
> +PHONY += sbom-report
> +sbom-report: $(RELEASEDIR)/sbom-report.json
> +
> +$(RELEASEDIR)/sbom-report.json: $(STATEDIR)/host-system-python3.install $(addprefix $(STATEDIR)/,$(addsuffix .fast-report,$(PACKAGES)))
> +	@$(call targetinfo)
> +	@$(call ptx/report-env, $@) ptxd_make_sbom_report
> +	@$(call finish)
> +

So I don't think this should be one target like this. The old license
documents work like this. But it's not very flexible. What you typically
want is basically an SBOM for an image with all its dependencies or maybe
one for the whole BSP (including all host packages).

The implementation should be flexible enough to do this. I've done that for
the 'new' license documents in scripts/generate-report.py and I am
currently extending that to create an SPDX SBOM.

And the input is always the full-bsp-report.yaml (or maybe
fast-bsp-report.yaml) but never the individual fragments.

>  # vim: syntax=make
> diff --git a/rules/sbom-report.in b/rules/sbom-report.in
> new file mode 100644
> index 000000000..311ad5a23
> --- /dev/null
> +++ b/rules/sbom-report.in
> @@ -0,0 +1,9 @@
> +## SECTION=ptxdist_options
> +
> +config ENABLE_SBOM_REPORT
> +	bool
> +	select HOST_SYSTEM_PYTHON3
> +	select HOST_SYSTEM_PYTHON3_CYCLONEDX
> +	prompt "enable sbom report generation"
> +	help
> +	  This enables the generation of an SBOM report, through ptxdist sbom-report.
> diff --git a/scripts/lib/ptxd_make_report.sh b/scripts/lib/ptxd_make_report.sh
> index 885f07828..f2e92e6be 100644
> --- a/scripts/lib/ptxd_make_report.sh
> +++ b/scripts/lib/ptxd_make_report.sh
> @@ -146,3 +146,19 @@ ptxd_make_fast_bsp_report() {
>  }
>  export -f ptxd_make_fast_bsp_report
>  
> +ptxd_make_sbom_report() {
> +    local -a ptxd_reply
> +    local pkg_lic pkg
> +
> +    ptxd_make_layer_init || return
> +
> +    echo "Generating $(ptxd_print_path "${ptx_report_target}") ..."
> +    echo
> +    
> +    mkdir -p "$(dirname "${ptx_report_target}")" &&
> +    "${ptx_system_python3}" ${PTXDIST_LIB_DIR}/ptxd_make_sbom_report.py "${ptx_report_dir}/fast/" ${ptx_target_packages} > ${PTXDIST_TEMPDIR}/sbom-report &&
> +    mv "${PTXDIST_TEMPDIR}/sbom-report" "${ptx_report_target}" ||
> +    ptxd_bailout "failed to create SBOM report"
> +}
> +export -f ptxd_make_sbom_report
> +
> diff --git a/scripts/lib/ptxd_make_sbom_report.py b/scripts/lib/ptxd_make_sbom_report.py
> new file mode 100644
> index 000000000..a9ec36697
> --- /dev/null
> +++ b/scripts/lib/ptxd_make_sbom_report.py
> @@ -0,0 +1,52 @@
> +from cyclonedx.factory.license import LicenseFactory
> +from cyclonedx.model.bom import Bom
> +from cyclonedx.model.component import Component
> +from cyclonedx.output.json import JsonV1Dot4
> +import sys
> +import re
> +
> +lFac = LicenseFactory()
> +bom = Bom()
> +
> +for i in range(2, len(sys.argv)):
> +    pkg_report = sys.argv[1] + sys.argv[i] + ".yaml"
> +    with open(pkg_report, 'r') as file:
> +        content = file.read()
> +        name_ = re.search("name: \'(.+)\'", content).group(1)
> +        version_ = re.search("version: \'(.+)\'", content).group(1)
> +
> +        # First see if we have a full CPE specified, then use that
> +        cpe_match = re.search("cpe: \'(.+)\'", content)
> +        cpe_ = None
> +        if cpe_match is not None:
> +            cpe_ = cpe_match.group(1)
> +        else:
> +            # See if we have the individual components
> +            cpe_vendor_match = re.search("cpe_vendor: \'(.+)\'", content)
> +            cpe_product_match = re.search("cpe_product: \'(.+)\'", content)
> +            cpe_version_match = re.search("cpe_version: \'(.+)\'", content)
> +            if cpe_vendor_match is not None and cpe_product_match is not None and cpe_version_match is not None:
> +                cpe_ = "cpe:2.3:a:{vendor}:{product}:{version}:*:*:*:*:*:*:*".format(vendor=cpe_vendor_match.group(1),
> +                                                                                     product=cpe_product_match.group(1),
> +                                                                                     version=cpe_version_match.group(1))
> +
> +        if cpe_ is not None:
> +            # We have a CPE, let's validate it. Regex from: https://csrc.nist.gov/schema/cpe/2.3/cpe-naming_2.3.xsd
> +            if not re.fullmatch("cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!\"#$$%&'\(\)\+,/:;"
> +                                "<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)"
> +                                "|[\*\-]))(:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!\"#$$%&'\(\)\+,/:;<=>@\[\]\^`\{\|}~]"
> +                                "))+(\?*|\*?))|[\*\-])){4}", cpe_):
> +                raise ValueError("Constructed CPE is not valid: {cpe}".format(cpe=cpe_))
> +
> +        licenses_ = re.search("licenses: \'(.+)\'", content).group(1)
> +        comp = Component(
> +            name=name_,
> +            version=version_,
> +            cpe=cpe_,
> +            licenses=[lFac.make_from_string(licenses_)],
> +            bom_ref=name_ + "@" + version_
> +        )
> +        bom.components.add(comp)
> +
> +serializedJSON = JsonV1Dot4(bom).output_as_string()
> +print(serializedJSON)

This needs to be quite different. We have one yaml file with all the data
and that should be used.

As I said, I'm working on an SPDX SBOM. It's probably best if you wait for
that to hit ptxdist master and build on top of that. That will probably
happen this or next week, at least the initial version.

Regards,
Michael


-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |

Re: [ptxdist] [PATCH 1/3] RFC: ptxd_make_world: Extract CPE for packages

[Simon Falsig]

Hi Christian,

> From: Christian Melki <christian.melki@t2data.com>
> Sent: Wednesday, September 13, 2023 23:17
> 
> On 9/13/23 18:05, Simon Falsig wrote:
> > From: Simon Falsig <sfalsig@verity.ch>
> >
> > If a package specifies a CPE, this is extracted into the fast report
> > for that package. If no CPE is specified, then no value is added.
> >
> > The CPE (Common Platform Enumerator) allows matching CVEs to specific
> > packages, and see if these apply to a specific deployment.
> 
> Hi Simon.
> 
> I think this is a good thing going forward, but some minor nag.
> My objection would be that sticking full versioned CPE strings straight
> into the .make as an only-source just creates clutter.
> As an full CPE override, absolutely though.
> 
> I suggest that some basic CPE modelling should be done by ptxdist, with
> possibly trivial hinting or nameing in the .make, with complete overrides
> as a last resort. That way ptxdist could start by filling most stuff and
> people could override on demand.
> 
> I'd primarily poke the vendor:product tuple. Maybe ptxdist could do
> packagename:packagename as default. If you specify the smaller override it
> could be something like APPL_CPE_VENDOR and APPL_CPE_PRODUCT. Here you
> could use * or other strings. Overriding any of them or both.
> APPL_CPE would serve as the full override.
> 
> That could help in hiding CPE format or other usages (subject to
> changes) in a lot of places. Hopefully, most packages won't require extra
> information to match.
> 
> Regards,
> Christian
> 

I fully agree that the complete CPE string in the makefile is not desirable.
When I was preparing the patch I was considering other ways, but in the end
decided to just start with the simplest, to avoid implementing too complex
logic before the first review...

Also, I was struggling to find the right tradeoff between granularity and
user-friendliness. I guess requiring either none or both of _CPE_VENDOR and
_CPE_PRODUCT could work (to reduce the risk of wrong CPEs due to incorrect
generic defaults - so only generate a CPE if both exist) - and then allowing
a full override if _CPE is specified.
In most cases that should be enough (for now at least), I think. I'll have
a look at an implementation one of the next days.

Thanks for the input!
- Simon

Re: [ptxdist] [PATCH 1/3] RFC: ptxd_make_world: Extract CPE for packages

[Christian Melki]

On 9/13/23 18:05, Simon Falsig wrote:
> From: Simon Falsig <sfalsig@verity.ch>
> 
> If a package specifies a CPE, this is extracted into the fast report for
> that package. If no CPE is specified, then no value is added.
> 
> The CPE (Common Platform Enumerator) allows matching CVEs to specific
> packages, and see if these apply to a specific deployment.

Hi Simon.

I think this is a good thing going forward, but some minor nag.
My objection would be that sticking full versioned CPE strings straight
into the .make as an only-source just creates clutter.
As an full CPE override, absolutely though.

I suggest that some basic CPE modelling should be done by ptxdist, with
possibly trivial hinting or nameing in the .make, with complete
overrides as a last resort. That way ptxdist could start by filling most
stuff and people could override on demand.

I'd primarily poke the vendor:product tuple. Maybe ptxdist could do
packagename:packagename as default. If you specify the smaller override
it could be something like APPL_CPE_VENDOR and APPL_CPE_PRODUCT. Here
you could use * or other strings. Overriding any of them or both.
APPL_CPE would serve as the full override.

That could help in hiding CPE format or other usages (subject to
changes) in a lot of places. Hopefully, most packages won't require
extra information to match.

Regards,
Christian

> ---
>  rules/post/ptxd_make_world_common.make | 1 +
>  scripts/lib/ptxd_make_world_report.sh  | 1 +
>  2 files changed, 2 insertions(+)
> 
> diff --git a/rules/post/ptxd_make_world_common.make b/rules/post/ptxd_make_world_common.make
> index 08120607a..6c646fb16 100644
> --- a/rules/post/ptxd_make_world_common.make
> +++ b/rules/post/ptxd_make_world_common.make
> @@ -78,6 +78,7 @@ world/env/impl = \
>  	pkg_PKG="$(call ptx/escape,$(1))"					\
>  	pkg_pkg="$(call ptx/escape,$($(1)))"					\
>  	pkg_version="$(call ptx/escape,$($(1)_VERSION))"			\
> +	pkg_cpe="$(call ptx/escape,$($(1)_CPE))"			\
>  	pkg_config="$(call ptx/escape,$($(1)_CONFIG))"				\
>  	pkg_ref_config="$(call ptx/escape,$($(1)_REF_CONFIG))"			\
>  	pkg_path="$(call ptx/escape,$($(1)_PATH))"				\
> diff --git a/scripts/lib/ptxd_make_world_report.sh b/scripts/lib/ptxd_make_world_report.sh
> index dbdae5736..2bfe4c201 100644
> --- a/scripts/lib/ptxd_make_world_report.sh
> +++ b/scripts/lib/ptxd_make_world_report.sh
> @@ -39,6 +39,7 @@ ptxd_make_world_report_yaml() {
>      do_list "rundeps:" "${pkg_run_deps}"
>      do_echo "config:" "${pkg_config}"
>      do_echo "version:" "${pkg_version}"
> +    do_echo "cpe:" "${pkg_cpe}"
>      do_list "url:" "${pkg_url}"
>      do_echo "md5:" "${pkg_md5}"
>      do_echo "source:" "${pkg_src}"

[ptxdist] [PATCH 1/3] RFC: ptxd_make_world: Extract CPE for packages

[Simon Falsig]

From: Simon Falsig <sfalsig@verity.ch>

If a package specifies a CPE, this is extracted into the fast report for
that package. If no CPE is specified, then no value is added.

The CPE (Common Platform Enumerator) allows matching CVEs to specific
packages, and see if these apply to a specific deployment.
---
 rules/post/ptxd_make_world_common.make | 1 +
 scripts/lib/ptxd_make_world_report.sh  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/rules/post/ptxd_make_world_common.make b/rules/post/ptxd_make_world_common.make
index 08120607a..6c646fb16 100644
--- a/rules/post/ptxd_make_world_common.make
+++ b/rules/post/ptxd_make_world_common.make
@@ -78,6 +78,7 @@ world/env/impl = \
 	pkg_PKG="$(call ptx/escape,$(1))"					\
 	pkg_pkg="$(call ptx/escape,$($(1)))"					\
 	pkg_version="$(call ptx/escape,$($(1)_VERSION))"			\
+	pkg_cpe="$(call ptx/escape,$($(1)_CPE))"			\
 	pkg_config="$(call ptx/escape,$($(1)_CONFIG))"				\
 	pkg_ref_config="$(call ptx/escape,$($(1)_REF_CONFIG))"			\
 	pkg_path="$(call ptx/escape,$($(1)_PATH))"				\
diff --git a/scripts/lib/ptxd_make_world_report.sh b/scripts/lib/ptxd_make_world_report.sh
index dbdae5736..2bfe4c201 100644
--- a/scripts/lib/ptxd_make_world_report.sh
+++ b/scripts/lib/ptxd_make_world_report.sh
@@ -39,6 +39,7 @@ ptxd_make_world_report_yaml() {
     do_list "rundeps:" "${pkg_run_deps}"
     do_echo "config:" "${pkg_config}"
     do_echo "version:" "${pkg_version}"
+    do_echo "cpe:" "${pkg_cpe}"
     do_list "url:" "${pkg_url}"
     do_echo "md5:" "${pkg_md5}"
     do_echo "source:" "${pkg_src}"
-- 
2.25.1