Merge branch 'feature/submodules_sbom' into 'master'

tools: add sbom information for submodules

Closes IDF-7416

See merge request espressif/esp-idf!23989

.gitlab/ci/host-test.yml

@@ -213,6 +213,14 @@ test_mkuf2:
     - cd ${IDF_PATH}/tools/test_mkuf2
     - ./test_mkuf2.py
 
+test_sbom:
+  extends:
+    - .host_test_template
+    - .rules:patterns:sbom
+  script:
+    - cd ${IDF_PATH}/tools/test_sbom
+    - pytest
+
 test_autocomplete:
   extends:
     - .host_test_template

.gitlab/ci/rules.yml

@@ -54,6 +54,9 @@
   - "tools/ci/ci_build_apps.py"
   - "tools/test_build_system/**/*"
 
+.patterns-sbom: &patterns-sbom
+  - "tools/test_sbom/*"
+
 .patterns-custom_test: &patterns-custom_test
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/tiny_test_fw/**/*"
@@ -445,6 +448,14 @@
     - <<: *if-dev-push
       changes: *patterns-sonarqube-files
 
+.rules:patterns:sbom:
+  rules:
+    - <<: *if-protected
+    - <<: *if-dev-push
+      changes: *patterns-sbom
+    - <<: *if-dev-push
+      changes: *patterns-submodule
+
 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 # DO NOT place comments or maintain any code from this line
 #

.gitmodules

@@ -2,6 +2,26 @@
 # All the relative URL paths are intended to be GitHub ones
 # For Espressif's public projects please use '../../espressif/proj', not a '../proj'
 #
+# Submodules SBOM information
+# ---------------------------
+# Submodules, which are used directly and not forked into espressif namespace should
+# contain SBOM information here. Other submodules should have the SBOM manifest file
+# included in the root of their project's repository.
+#
+# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree
+# commit object. For example spiffs submodule
+#
+# $ git ls-tree HEAD components/spiffs/spiffs
+# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852	components/spiffs/spiffs
+#
+# The hash can be also obtained with git submodule command
+#
+# $ git submodule status components/spiffs/spiffs
+# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6)
+#
+# The submodule SHA recorded here has to match with SHA, which is presented in git-tree.
+# This is checked by CI. Also please don't forget to update the submodule version
+# if you are changing the sbom-hash. This is important for SBOM generation.
 
 [submodule "components/bt/controller/lib_esp32"]
 	path = components/bt/controller/lib_esp32
@@ -10,14 +30,31 @@
 [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
 	path = components/bootloader/subproject/components/micro-ecc/micro-ecc
 	url = ../../kmackay/micro-ecc.git
+	sbom-version = 1.0
+	sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Ken MacKay
+	sbom-url = https://github.com/kmackay/micro-ecc
+	sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors
+	sbom-hash = d037ec89546fad14b5c4d5456c2e23a71e554966
 
 [submodule "components/spiffs/spiffs"]
 	path = components/spiffs/spiffs
 	url = ../../pellepl/spiffs.git
+	sbom-version = 0.2-255-g0dbb3f71c5f6
+	sbom-supplier = Person: Peter Andersson
+	sbom-url = https://github.com/pellepl/spiffs
+	sbom-description = Wear-leveled SPI flash file system for embedded devices
+	sbom-hash = 0dbb3f71c5f6fae3747a9d935372773762baf852
 
 [submodule "components/json/cJSON"]
 	path = components/json/cJSON
 	url = ../../DaveGamble/cJSON.git
+	sbom-version = 1.7.15
+	sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Dave Gamble
+	sbom-url = https://github.com/DaveGamble/cJSON
+	sbom-description = Ultralightweight JSON parser in ANSI C
+	sbom-hash = d348621ca93571343a56862df7de4ff3bc9b5667
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls
@@ -34,10 +71,21 @@
 [submodule "components/protobuf-c/protobuf-c"]
 	path = components/protobuf-c/protobuf-c
 	url = ../../protobuf-c/protobuf-c.git
+	sbom-version = 1.4.1
+	sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:*
+	sbom-supplier = Organization: protobuf-c community <https://groups.google.com/g/protobuf-c>
+	sbom-url = https://github.com/protobuf-c/protobuf-c
+	sbom-description = Protocol Buffers implementation in C
+	sbom-hash = abc67a11c6db271bedbb9f58be85d6f4e2ea8389
 
 [submodule "components/unity/unity"]
 	path = components/unity/unity
 	url = ../../ThrowTheSwitch/Unity.git
+	sbom-version = v2.4.3-51-g7d2bf62b7e6a
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/Unity
+	sbom-description = Simple Unit Testing for C
+	sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25
 
 [submodule "components/bt/host/nimble/nimble"]
 	path = components/bt/host/nimble/nimble
@@ -50,6 +98,11 @@
 [submodule "components/cmock/CMock"]
 	path = components/cmock/CMock
 	url = ../../ThrowTheSwitch/CMock.git
+	sbom-version = v2.5.2-2-geeecc49ce8af
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/CMock
+	sbom-description = CMock - Mock/stub generator for C
+	sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f
 
 [submodule "components/openthread/openthread"]
 	path = components/openthread/openthread

tools/test_build_system/pytest.ini

@@ -1,5 +1,4 @@
 [pytest]
-addopts = -s -p no:pytest-embedded
 
 # log related
 log_cli = True

tools/test_sbom/pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+addopts = -s -p no:pytest_embedded
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+## log all to `system-out` when case fail
+junit_logging = stdout
+junit_log_passing_tests = False

tools/test_sbom/test_submodules.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import os
+from subprocess import run
+from typing import Dict, List
+
+
+def run_cmd(cmd: List[str]) -> str:
+    """Simple helper to run command and return it's stdout."""
+    proc = run(cmd, capture_output=True, check=True, text=True)
+    return proc.stdout.strip()
+
+
+def get_gitwdir() -> str:
+    """Return absolute path to the current git working tree."""
+    return run_cmd(['git', 'rev-parse', '--show-toplevel'])
+
+
+def get_submodules_info() -> List[Dict[str,str]]:
+    """Return list of submodules, where each submodule is represented
+    as dictionary with name, path and hash keys."""
+    cmd = ['git', 'submodule', '--quiet', 'foreach','echo "$name,$sm_path,$sha1"']
+    out = run_cmd(cmd)
+    submodules = []
+    for line in out.splitlines():
+        name, sm_path, sha1 = line.split(',')
+        submodules += [{'name': name, 'path': sm_path, 'hash': sha1}]
+
+    return submodules
+
+
+def get_submodules_config() -> Dict[str,str]:
+    """Return dictionary, where key is variable name and value
+    is variable value in git's --list(dot) format. Only variables
+    starting with "submodule." are returned and this prefix is removed
+    to make it simple to match against the submodule info dictionary."""
+    gitmodules_fn = os.path.join(get_gitwdir(), '.gitmodules')
+    gitmodules_data = run_cmd(['git', 'config', '--list', '--file', gitmodules_fn])
+    prefix = 'submodule.'
+    config = {}
+    for line in gitmodules_data.splitlines():
+        var, val = line.split('=', maxsplit=1)
+        if not var.startswith(prefix):
+            continue
+        # remove "submodule." prefix
+        var = var[len(prefix):]
+        config[var] = val
+
+    return config
+
+
+def test_sha() -> None:
+    """ Check that submodule SHA1 in git-tree and .gitmodules match
+    if sbom-hash variable is available in the .gitmodules file.
+    """
+    submodules = get_submodules_info()
+    config = get_submodules_config()
+
+    for submodule in submodules:
+        sbom_hash = config.get(submodule['name'] + '.sbom-hash')
+        if not sbom_hash:
+            continue
+        msg = (f'Submodule \"{submodule["name"]}\" SHA \"{submodule["hash"]}\" in git '
+               f'tree does not match SHA \"{sbom_hash}\" recorded in .gitmodules. '
+               f'Please update \"sbom-hash\" in .gitmodules for \"{submodule["name"]}\" '
+               f'and also please do not forget to update version and other submodule '
+               f'information if necessary. It is important to keep this information '
+               f'up-to-date for SBOM generation.')
+        assert submodule['hash'] == sbom_hash, msg