mirror of https://github.com/mongodb/mongo
SERVER-114494 Improvements to SBOM automation [v7.0-staging] (#44603)
GitOrigin-RevId: cc2857a89a74018006e0f4ed91056505c09dcfbe
This commit is contained in:
Jason Hills
MongoDB Bot
parent
460abcf007
commit
2d7d5f44a9
|
|
@ -1,4 +1,4 @@
|
|||
[DO NOT MODIFY THIS FILE MANUALLY. It is generated by src/third_party/tools/gen_thirdpartyreadme.py]: #
|
||||
[DO NOT MODIFY THIS FILE MANUALLY. It is generated by src/third_party/scripts/gen_thirdpartyreadme.py]: #
|
||||
|
||||
# MongoDB Third Party Dependencies
|
||||
|
||||
|
|
@ -25,121 +25,87 @@ a notice will be included in
|
|||
| ---------------------------------------------------- | --------------------------------- | ---------------------------------------- | -------------------- | ------------------------------- |
|
||||
| [Abseil Common Libraries (C++)] | Apache-2.0 | 20211102.0 | | ✗ |
|
||||
| [Asio C++ Library] | BSL-1.0 | 1.12.2 | | ✗ |
|
||||
| [benchmark] | Apache-2.0 | v1.5.2 | | |
|
||||
| [Boost C++ Libraries] | BSL-1.0 | 1.79.0 | | ✗ |
|
||||
| [c-ares] | MIT | 1.19.1 | | |
|
||||
| [Cyrus SASL] | BSD-Attribution-HPND-disclaimer | 2.1.28 | | |
|
||||
| [fmt] | MIT | 7.1.3 | | ✗ |
|
||||
| [github.com/facebook/folly] | Apache-2.0 | v2025.04.21.00 | | ✗ |
|
||||
| [gperftools] | BSD-3-Clause | 2.9.1 | | ✗ |
|
||||
| [gRPC (C++)] | Apache-2.0 | 1.46.6 | | |
|
||||
| [immer] | BSL-1.0 | 0.8.0 | | ✗ |
|
||||
| [Intel® Decimal Floating-Point Math Library] | BSD-3-Clause | v2.0U1 | | ✗ |
|
||||
| [International Components for Unicode C/C++ (ICU4C)] | Unicode-3.0 | 57.1 | ✗ | ✗ |
|
||||
| [ICU4C - International Components for Unicode C/C++] | Unicode-3.0 | 57.1 | ✗ | ✗ |
|
||||
| [Intel® Decimal Floating-Point Math Library] | BSD-3-Clause | 2.0.1 | | ✗ |
|
||||
| [JSON Schema Store] | Apache-2.0 | 6847cfc3a17a04a7664474212db50c627e1e3408 | | |
|
||||
| [JSON-Schema-Test-Suite] | MIT | 728066f9c5c258ba3b1804a22a5b998f2ec77ec0 | | |
|
||||
| [libmongocrypt] | Apache-2.0 | e656245b7ebc742df210c8156b9aac41bdd5d113 | ✗ | ✗ |
|
||||
| [librdkafka - The Apache Kafka C/C++ library] | BSD-2-Clause | 2.0.2 | | ✗ |
|
||||
| [LibTomCrypt] | Unlicense | 1.18.2 | ✗ | ✗ |
|
||||
| [libunwind] | MIT | v1.6.2 | | ✗ |
|
||||
| [linenoise] | BSD-2-Clause | 6cdc775807e57b2c3fd64bd207814f8ee1fe35f3 | | ✗ |
|
||||
| [MongoDB C Driver] | Apache-2.0 | 1.27.6 | ✗ | ✗ |
|
||||
| [Mozilla Firefox ESR] | MPL-2.0 | 128.11.0esr | | ✗ |
|
||||
| [MurmurHash3] | Public Domain | a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb | | ✗ |
|
||||
| [nlohmann/json] | MIT | 3.10.5 | | |
|
||||
| [PCRE2 - Perl-Compatible Regular Expressions] | BSD-3-Clause WITH PCRE2-exception | 10.40 | | ✗ |
|
||||
| [Protobuf] | BSD-3-Clause | v3.19.5 | | ✗ |
|
||||
| [pypi/asn1crypto] | MIT | 1.5.1 | | |
|
||||
| [pypi/concurrencytest] | GPL-3.0-or-later | 0.1.2 | | |
|
||||
| [pypi/discover] | BSD-3-Clause | 0.4.0 | | |
|
||||
| [pypi/extras] | MIT | 0.0.3 | | |
|
||||
| [Protobuf] | BSD-3-Clause | 3.19.5 | | ✗ |
|
||||
| [S2 Geometry Library] | Apache-2.0 | a25c502bda9d7e0274b9e2b7825fbddf13cc0306 | ✗ | ✗ |
|
||||
| [SCons - a Software Construction tool] | MIT | 4.9.1 | | |
|
||||
| [SafeInt] | MIT | 3.0.26 | | ✗ |
|
||||
| [Snowball Stemming Algorithms (libstemmer)] | BSD-3-Clause | 1.0.0 | ✗ | ✗ |
|
||||
| [Unicode Character Database] | Unicode-DFS-2016 | 8.0.0 | ✗ | ✗ |
|
||||
| [WiredTiger] | GPL-2.0-only OR GPL-3.0-only | 11.2.0 | ✗ | ✗ |
|
||||
| [Zstandard (zstd)] | BSD-3-Clause OR GPL-2.0-only | 1.5.2 | ✗ | ✗ |
|
||||
| [benchmark] | Apache-2.0 | 1.5.2 | | |
|
||||
| [c-ares] | MIT | 1.19.1 | | ✗ |
|
||||
| [fmt] | MIT | 7.1.3 | | ✗ |
|
||||
| [folly] | Apache-2.0 | 2025.04.21.00 | | ✗ |
|
||||
| [gRPC (C++)] | Apache-2.0 | 1.46.6 | | ✗ |
|
||||
| [gperftools] | BSD-3-Clause | 2.9.1 | | ✗ |
|
||||
| [immer] | BSL-1.0 | 0b3aaf699b9d6f2e89f8e2b6d1221c307e02bda3 | | ✗ |
|
||||
| [libmongocrypt] | Apache-2.0 | e656245b7ebc742df210c8156b9aac41bdd5d113 | ✗ | ✗ |
|
||||
| [librdkafka - The Apache Kafka C/C++ library] | BSD-2-Clause | 2.0.2 | | ✗ |
|
||||
| [libunwind] | MIT | 1.6.1 | | ✗ |
|
||||
| [linenoise] | BSD-2-Clause | 6cdc775807e57b2c3fd64bd207814f8ee1fe35f3 | | ✗ |
|
||||
| [pypi/ocspbuilder] | MIT | 0.10.2 | | |
|
||||
| [pypi/ocspresponder] | Apache-2.0 | 0.5.0 | | |
|
||||
| [pypi/oscrypto] | MIT | 1.3.0 | | |
|
||||
| [pypi/python-subunit] | (Apache-2.0 OR BSD-3-Clause) | 0.0.16 | | |
|
||||
| [pypi/testscenarios] | BSD-3-Clause | 0.4 | | |
|
||||
| [pypi/testtools] | MIT | 0.9.34 | | |
|
||||
| [re2] | BSD-3-Clause | 2021-09-01 | | |
|
||||
| [S2 Geometry Library] | Apache-2.0 | c872048da5d1 | ✗ | ✗ |
|
||||
| [SafeInt] | MIT | 3.0.26 | | ✗ |
|
||||
| [SCons - a Software Construction tool] | MIT | 4.9.1 | | |
|
||||
| [re2] | BSD-3-Clause | 2021-09-01 | | ✗ |
|
||||
| [snappy] | BSD-3-Clause | 1.1.7 | ✗ | ✗ |
|
||||
| [Snowball Stemming Algorithms (libstemmer)] | BSD-3-Clause | 7b264ffa0f767c579d052fd8142558dc8264d795 | ✗ | ✗ |
|
||||
| [timelib] | MIT | 2022.10 | | ✗ |
|
||||
| [Unicode Character Database] | Unicode-DFS-2016 | 8.0.0 | ✗ | ✗ |
|
||||
| [valgrind.h] | BSD-4-Clause | 3.17.0 | | ✗ |
|
||||
| [variant] | BSL-1.0 | v1.4.0 | unknown | ✗ |
|
||||
| [WiredTiger] | GPL-2.0-only OR GPL-3.0-only | mongodb-7.0.22 | ✗ | ✗ |
|
||||
| [valgrind.h] | BSD-4-Clause | 093bef43d69236287ccc748591c9560a71181b0a | | ✗ |
|
||||
| [variant] | BSL-1.0 | v1.4.0 | | ✗ |
|
||||
| [yaml-cpp] | MIT | 0.6.3 | | ✗ |
|
||||
| [zlib] | Zlib | 1.2.13 | ✗ | ✗ |
|
||||
| [Zstandard (zstd)] | BSD-3-Clause OR GPL-2.0-only | 1.5.2 | ✗ | ✗ |
|
||||
|
||||
[Abseil Common Libraries (C++)]: https://github.com/abseil/abseil-cpp
|
||||
[Asio C++ Library]: https://github.com/chriskohlhoff/asio
|
||||
[Boost C++ Libraries]: http://www.boost.org/
|
||||
[Cyrus SASL]: https://www.cyrusimap.org/sasl/
|
||||
[Intel® Decimal Floating-Point Math Library]: https://software.intel.com/en-us/articles/intel-decimal-floating-point-math-library
|
||||
[International Components for Unicode C/C++ (ICU4C)]: http://site.icu-project.org/download/
|
||||
[JSON Schema Store]: https://www.schemastore.org/json/
|
||||
[JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite
|
||||
[LibTomCrypt]: https://github.com/libtom/libtomcrypt/releases
|
||||
[MongoDB C Driver]: https://github.com/mongodb/mongo-c-driver
|
||||
[Mozilla Firefox ESR]: https://www.mozilla.org/en-US/security/known-vulnerabilities/firefox-esr
|
||||
[Abseil Common Libraries (C++)]: https://github.com/abseil/abseil-cpp.git
|
||||
[Asio C++ Library]: https://github.com/chriskohlhoff/asio.git
|
||||
[Boost C++ Libraries]: https://github.com/boostorg/boost.git
|
||||
[Cyrus SASL]: https://github.com/cyrusimap/cyrus-sasl.git
|
||||
[ICU4C - International Components for Unicode C/C++]: https://github.com/unicode-org/icu.git
|
||||
[Intel® Decimal Floating-Point Math Library]: https://www.netlib.org/misc/intel/
|
||||
[JSON Schema Store]: https://github.com/schemastore/schemastore.git
|
||||
[JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite.git
|
||||
[LibTomCrypt]: https://github.com/libtom/libtomcrypt.git
|
||||
[MongoDB C Driver]: https://github.com/mongodb/mongo-c-driver.git
|
||||
[Mozilla Firefox ESR]: https://github.com/mozilla-firefox/firefox.git
|
||||
[MurmurHash3]: https://github.com/aappleby/smhasher/blob/a6bd3ce/
|
||||
[PCRE2 - Perl-Compatible Regular Expressions]: http://www.pcre.org/
|
||||
[Protobuf]: https://github.com/protocolbuffers/protobuf
|
||||
[S2 Geometry Library]: https://github.com/google/s2geometry
|
||||
[SCons - a Software Construction tool]: https://github.com/SCons/scons
|
||||
[SafeInt]: https://github.com/dcleblanc/SafeInt
|
||||
[Snowball Stemming Algorithms (libstemmer)]: https://github.com/snowballstem/snowball
|
||||
[Unicode Character Database]: http://www.unicode.org/versions/enumeratedversions.html
|
||||
[WiredTiger]: https://source.wiredtiger.com/
|
||||
[Zstandard (zstd)]: https://github.com/facebook/zstd
|
||||
[benchmark]: https://github.com/google/benchmark
|
||||
[c-ares]: https://c-ares.org/
|
||||
[fmt]: http://fmtlib.net/
|
||||
[gRPC (C++)]: https://github.com/grpc/grpc
|
||||
[github.com/facebook/folly]: https://github.com/facebook/folly
|
||||
[gperftools]: https://github.com/gperftools/gperftools
|
||||
[immer]: https://github.com/arximboldi/immer
|
||||
[libmongocrypt]: https://github.com/mongodb/libmongocrypt
|
||||
[librdkafka - The Apache Kafka C/C++ library]: https://github.com/confluentinc/librdkafka
|
||||
[libunwind]: http://www.github.com/libunwind/libunwind
|
||||
[PCRE2 - Perl-Compatible Regular Expressions]: https://github.com/pcre2project/pcre2.git
|
||||
[Protobuf]: https://github.com/protocolbuffers/protobuf.git
|
||||
[S2 Geometry Library]: https://github.com/google/s2geometry.git
|
||||
[SCons - a Software Construction tool]: https://github.com/scons/scons.git
|
||||
[SafeInt]: https://github.com/dcleblanc/safeint.git
|
||||
[Snowball Stemming Algorithms (libstemmer)]: http://github.com/snowballstem/snowball.git
|
||||
[Unicode Character Database]: https://www.unicode.org/Public/8.0.0/
|
||||
[WiredTiger]: https://github.com/wiredtiger/wiredtiger.git
|
||||
[Zstandard (zstd)]: https://github.com/facebook/zstd.git
|
||||
[benchmark]: https://github.com/google/benchmark.git
|
||||
[c-ares]: https://github.com/c-ares/c-ares.git
|
||||
[fmt]: https://github.com/fmtlib/fmt.git
|
||||
[folly]: https://github.com/facebook/folly.git
|
||||
[gRPC (C++)]: https://github.com/grpc/grpc.git
|
||||
[gperftools]: https://github.com/gperftools/gperftools.git
|
||||
[immer]: https://github.com/arximboldi/immer.git
|
||||
[libmongocrypt]: https://github.com/mongodb/libmongocrypt.git
|
||||
[librdkafka - The Apache Kafka C/C++ library]: https://github.com/confluentinc/librdkafka.git
|
||||
[libunwind]: https://github.com/libunwind/libunwind.git
|
||||
[linenoise]: https://github.com/antirez/linenoise
|
||||
[nlohmann/json]: https://github.com/nlohmann/json
|
||||
[pypi/asn1crypto]: https://github.com/wbond/asn1crypto
|
||||
[pypi/concurrencytest]: https://pypi.org/project/concurrencytest/
|
||||
[pypi/discover]: https://pypi.org/project/discover/
|
||||
[pypi/extras]: https://github.com/testing-cabal/extras
|
||||
[pypi/ocspbuilder]: https://github.com/wbond/ocspbuilder
|
||||
[pypi/ocspresponder]: https://github.com/threema-ch/ocspresponder
|
||||
[pypi/oscrypto]: https://github.com/wbond/oscrypto
|
||||
[pypi/python-subunit]: https://github.com/testing-cabal/subunit
|
||||
[pypi/testscenarios]: https://pypi.org/project/testscenarios/
|
||||
[pypi/testtools]: https://github.com/testing-cabal/testtools
|
||||
[re2]: https://github.com/google/re2
|
||||
[snappy]: https://github.com/google/snappy/releases
|
||||
[timelib]: https://github.com/derickr/timelib
|
||||
[valgrind.h]: http://valgrind.org/downloads/current.html
|
||||
[variant]: https://github.com/mpark/variant
|
||||
[yaml-cpp]: https://github.com/jbeder/yaml-cpp/releases
|
||||
[zlib]: https://zlib.net/
|
||||
|
||||
## WiredTiger Vendored Test Libraries
|
||||
|
||||
The following libraries are transitively included by WiredTiger,
|
||||
and are used by that component for testing. They don't appear in
|
||||
released binary artifacts.
|
||||
|
||||
| Name |
|
||||
| -------------------------- |
|
||||
| nlohmann/json@3.10.5 |
|
||||
| pypi/concurrencytest@0.1.2 |
|
||||
| pypi/discover@0.4.0 |
|
||||
| pypi/extras@0.0.3 |
|
||||
| pypi/python-subunit@0.0.16 |
|
||||
| pypi/testscenarios@0.4 |
|
||||
| pypi/testtools@0.9.34 |
|
||||
[pypi/ocspbuilder]: https://pypi.org/project/ocspbuilder/
|
||||
[pypi/ocspresponder]: https://pypi.org/project/ocspresponder/
|
||||
[re2]: https://github.com/google/re2.git
|
||||
[snappy]: https://github.com/google/tcmalloc.git
|
||||
[timelib]: https://github.com/derickr/timelib.git
|
||||
[valgrind.h]: https://sourceware.org/git/valgrind.git
|
||||
[variant]: https://github.com/mpark/variant.git
|
||||
[yaml-cpp]: https://github.com/jbeder/yaml-cpp.git
|
||||
[zlib]: https://zlib.net/fossils/
|
||||
|
||||
## Dynamically Linked Libraries
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
"""generate_sbom.py config. Operational configuration values stored separately from the core code."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
|
|
@ -52,55 +51,6 @@ endor_components_rename = [
|
|||
["pkg:c/github.com/", "pkg:github/"],
|
||||
]
|
||||
|
||||
# ################ PURL Validation ################
|
||||
REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #)
|
||||
r"(?:@[^?@#]*)?"
|
||||
# Optional Qualifiers (any chars except @ #)
|
||||
r"(?:\?[^@#]*)?"
|
||||
# Optional Subpath (any chars)
|
||||
r"(?:#.*)?$")
|
||||
|
||||
REGEX_PURL = {
|
||||
# deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md
|
||||
"deb":
|
||||
re.compile(r"^pkg:deb/" # Scheme and type
|
||||
# Namespace (organization/user), letters must be lowercase
|
||||
r"(debian|ubuntu)+"
|
||||
r"/"
|
||||
r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name
|
||||
),
|
||||
# Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md
|
||||
"generic":
|
||||
re.compile(r"^pkg:generic/" # Scheme and type
|
||||
r"([a-zA-Z0-9._-]+/)?" # Optional namespace segment
|
||||
r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (required)
|
||||
),
|
||||
# GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md
|
||||
"github":
|
||||
re.compile(r"^pkg:github/" # Scheme and type
|
||||
# Namespace (organization/user), letters must be lowercase
|
||||
r"[a-z0-9-]+"
|
||||
r"/"
|
||||
r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (repository)
|
||||
),
|
||||
# PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md
|
||||
"pypi":
|
||||
re.compile(r"^pkg:pypi/" # Scheme and type
|
||||
r"[a-z0-9_-]+" # Name, letters must be lowercase, dashes, underscore
|
||||
+ REGEX_STR_PURL_OPTIONAL),
|
||||
}
|
||||
|
||||
|
||||
def is_valid_purl(purl: str) -> bool:
|
||||
"""Validate a GitHub or Generic PURL."""
|
||||
for purl_type, regex in REGEX_PURL.items():
|
||||
if regex.match(purl):
|
||||
logger.debug("PURL: %s matched PURL type '%s' regex '%s'", purl, purl_type,
|
||||
regex.pattern)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ################ Version Transformation ################
|
||||
|
||||
# In some cases we need to transform the version string to strip out tag-related text
|
||||
|
|
@ -119,6 +69,7 @@ VERSION_PATTERN_REPL = [
|
|||
# 'mongodb-8.2.0-alpha2' pkg:github/wiredtiger/wiredtiger
|
||||
# 'release-1.12.0' pkg:github/apache/avro
|
||||
# 'yaml-cpp-0.6.3' pkg:github/jbeder/yaml-cpp
|
||||
# 'node-v2.6.0' pkg:github/mongodb/libmongocrypt
|
||||
[re.compile(rf"^[-a-z]+[-/][vr]?({RE_SEMVER})$"), r"\1"],
|
||||
# 'asio-1-34-2' pkg:github/chriskohlhoff/asio
|
||||
# 'cares-1_27_0' pkg:github/c-ares/c-ares
|
||||
|
|
@ -150,17 +101,26 @@ def get_semver_from_release_version(release_ver: str) -> str:
|
|||
# region special component use-case functions
|
||||
|
||||
|
||||
def get_version_from_wiredtiger_import_data(file_path: str) -> str:
|
||||
"""Get the info in the 'import.data' file saved in the wiredtiger folder."""
|
||||
def get_version_from_wiredtiger_release_info(wt_dir: str) -> str:
|
||||
"""Get version from 'RELEASE_INFO' file in the wiredtiger folder."""
|
||||
|
||||
import os
|
||||
|
||||
ver = {}
|
||||
try:
|
||||
with open(file_path, "r") as input_json:
|
||||
import_data = input_json.read()
|
||||
result = json.loads(import_data)
|
||||
for line in open(os.path.join(wt_dir, "RELEASE_INFO"), "r", encoding="utf-8"):
|
||||
if re.match(r"WIREDTIGER_VERSION_(?:MAJOR|MINOR|PATCH)=", line):
|
||||
exec(line, ver)
|
||||
wt_ver = "%d.%d.%d" % (
|
||||
ver["WIREDTIGER_VERSION_MAJOR"],
|
||||
ver["WIREDTIGER_VERSION_MINOR"],
|
||||
ver["WIREDTIGER_VERSION_PATCH"],
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error("Error loading JSON file from %s", file_path)
|
||||
logger.error("Error loading file from %s", wt_dir)
|
||||
logger.error(err)
|
||||
return ""
|
||||
return result.get("commit")
|
||||
return wt_ver
|
||||
|
||||
|
||||
def get_version_sasl_from_workspace(file_path: str) -> str:
|
||||
|
|
@ -168,7 +128,7 @@ def get_version_sasl_from_workspace(file_path: str) -> str:
|
|||
# e.g.,
|
||||
# SASL_URL = "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
|
||||
try:
|
||||
with open(file_path, "r") as file:
|
||||
with open(file_path, "r", encoding="utf-8") as file:
|
||||
for line in file:
|
||||
if line.strip().startswith(
|
||||
'SASL_URL = "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-'
|
||||
|
|
@ -202,9 +162,9 @@ def process_component_special_cases(component_key: str, component_dict: dict, ve
|
|||
occurrences = component_dict.get("evidence", {}).get("occurrences", [])
|
||||
if occurrences:
|
||||
location = occurrences[0].get("location")
|
||||
versions["import_script"] = get_version_from_wiredtiger_import_data(
|
||||
f"{repo_root}/{location}/import.data")
|
||||
logger.info("VERSION SPECIAL CASE: %s: Found version '%s' in 'import.data' file",
|
||||
versions["import_script"] = get_version_from_wiredtiger_release_info(
|
||||
f"{repo_root}/{location}")
|
||||
logger.info("VERSION SPECIAL CASE: %s: Found version '%s' in 'RELEASE_INFO' file",
|
||||
component_key, versions['import_script'])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,13 +24,13 @@ from config import (
|
|||
endor_components_remove,
|
||||
endor_components_rename,
|
||||
get_semver_from_release_version,
|
||||
is_valid_purl,
|
||||
process_component_special_cases,
|
||||
)
|
||||
from endorctl_utils import EndorCtl
|
||||
from git import Commit, Repo
|
||||
|
||||
# region init
|
||||
# pylint: disable=W1201,W1203
|
||||
|
||||
|
||||
class WarningListHandler(logging.Handler):
|
||||
|
|
@ -66,6 +66,62 @@ REGEX_GITHUB_URL = r"^(https://github.com/)([a-zA-Z0-9-]{1,39}/[a-zA-Z0-9-_.]{1,
|
|||
REGEX_RELEASE_BRANCH = r"^v\d\.\d$"
|
||||
REGEX_RELEASE_TAG = r"^r\d\.\d.\d(-\w*)?$"
|
||||
|
||||
# ################ PURL Validation ################
|
||||
REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #)
|
||||
r"(?:@[^?@#]*)?"
|
||||
# Optional Qualifiers (any chars except @ #)
|
||||
r"(?:\?[^@#]*)?"
|
||||
# Optional Subpath (any chars)
|
||||
r"(?:#.*)?$")
|
||||
|
||||
REGEX_PURL = {
|
||||
# deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md
|
||||
"deb":
|
||||
re.compile(r"^pkg:deb/" # Scheme and type
|
||||
# Namespace (organization/user), letters must be lowercase
|
||||
r"(debian|ubuntu)+"
|
||||
r"/"
|
||||
r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name
|
||||
),
|
||||
# Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md
|
||||
"generic":
|
||||
re.compile(r"^pkg:generic/" # Scheme and type
|
||||
r"([a-zA-Z0-9._-]+/)?" # Optional namespace segment
|
||||
r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (required)
|
||||
),
|
||||
# GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md
|
||||
"github":
|
||||
re.compile(r"^pkg:github/" # Scheme and type
|
||||
# Namespace (organization/user), letters must be lowercase
|
||||
r"[a-z0-9-]+"
|
||||
r"/"
|
||||
r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (repository)
|
||||
),
|
||||
# PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md
|
||||
"pypi":
|
||||
re.compile(r"^pkg:pypi/" # Scheme and type
|
||||
r"[a-z0-9_-]+" # Name, letters must be lowercase, dashes, underscore
|
||||
+ REGEX_STR_PURL_OPTIONAL),
|
||||
}
|
||||
|
||||
# Metadata SBOM requirements
|
||||
METADATA_FIELDS_REQUIRED = [
|
||||
"type",
|
||||
"bom-ref",
|
||||
"group",
|
||||
"name",
|
||||
"version",
|
||||
"description",
|
||||
"licenses",
|
||||
"copyright",
|
||||
"externalReferences",
|
||||
"scope",
|
||||
]
|
||||
METADATA_FIELDS_ONE_OF = [
|
||||
["author", "supplier"],
|
||||
["purl", "cpe"],
|
||||
]
|
||||
|
||||
# endregion init
|
||||
|
||||
# region functions and classes
|
||||
|
|
@ -78,8 +134,13 @@ class GitInfo:
|
|||
print_banner("Gathering git info")
|
||||
try:
|
||||
self.repo_root = Path(
|
||||
subprocess.run("git rev-parse --show-toplevel", shell=True, text=True,
|
||||
capture_output=True).stdout.strip())
|
||||
subprocess.run(
|
||||
"git rev-parse --show-toplevel",
|
||||
shell=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
).stdout.strip())
|
||||
self._repo = Repo(self.repo_root)
|
||||
except Exception as ex:
|
||||
logger.warning(
|
||||
|
|
@ -168,6 +229,16 @@ def extract_repo_from_git_url(git_url: str) -> dict:
|
|||
}
|
||||
|
||||
|
||||
def is_valid_purl(purl: str) -> bool:
|
||||
"""Validate a GitHub or Generic PURL."""
|
||||
for purl_type, regex in REGEX_PURL.items():
|
||||
if regex.match(purl):
|
||||
logger.debug("PURL: %s matched PURL type '%s' regex '%s'", purl, purl_type,
|
||||
regex.pattern)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict:
|
||||
"""Create a dict of SBOM components with a version-less PURL as the key."""
|
||||
components = sbom["components"]
|
||||
|
|
@ -184,6 +255,22 @@ def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict:
|
|||
return components_dict
|
||||
|
||||
|
||||
def check_metadata_sbom(meta_bom: dict) -> None:
|
||||
"""Run checks on SBOM component metadata for expected fields."""
|
||||
for component in meta_bom["components"]:
|
||||
for field in METADATA_FIELDS_REQUIRED:
|
||||
if field not in component:
|
||||
logger.warning("METADATA: '%s' is missing required field '%s'.",
|
||||
component['bom-ref'] or component['name'], field)
|
||||
for fields in METADATA_FIELDS_ONE_OF:
|
||||
found = False
|
||||
for field in fields:
|
||||
found = found or field in component
|
||||
if not found:
|
||||
logger.warning("METADATA: '%s' is missing one of fields '%s'.", component['bom-ref']
|
||||
or component['name'], fields)
|
||||
|
||||
|
||||
def read_sbom_json_file(file_path: str) -> dict:
|
||||
"""Load a JSON SBOM file (schema is not validated)."""
|
||||
try:
|
||||
|
|
@ -204,8 +291,8 @@ def write_sbom_json_file(sbom_dict: dict, file_path: str) -> None:
|
|||
try:
|
||||
file_path = os.path.abspath(file_path)
|
||||
with open(file_path, "w", encoding="utf-8") as output_json:
|
||||
json.dump(sbom_dict, output_json, indent=2)
|
||||
output_json.write("\n")
|
||||
formatted_sbom = json.dumps(sbom_dict, indent=2) + "\n"
|
||||
output_json.write(formatted_sbom)
|
||||
except Exception as err:
|
||||
logger.error("Error writing SBOM file to %s", file_path)
|
||||
logger.error(err)
|
||||
|
|
@ -298,6 +385,32 @@ def get_component_import_script_path(component: dict) -> str:
|
|||
return ""
|
||||
|
||||
|
||||
def get_component_priority_version_source(component: dict) -> str:
|
||||
"""Get the priority version source, if defined in metadata file."""
|
||||
priority_version_source = [
|
||||
p.get("value") for p in component.get("properties", [])
|
||||
if p.get("name") == "generate_sbom:priority_version_source"
|
||||
]
|
||||
if len(priority_version_source):
|
||||
# There should only be 1 result, if any
|
||||
return priority_version_source[0]
|
||||
else:
|
||||
return ""
|
||||
|
||||
|
||||
def del_component_priority_version_source(component: dict) -> None:
|
||||
"""Delete all priority version source properties."""
|
||||
|
||||
# Reverse iterate properties list to safely modify in situ
|
||||
if "properties" in component:
|
||||
for i in range(len(component["properties"]) - 1, -1, -1):
|
||||
if component["properties"][i].get("name") == "generate_sbom:priority_version_source":
|
||||
logger.debug(
|
||||
"PRIORITY VERSION SOURCE: %s: Removing priority version source from SBOM metadata.",
|
||||
component['bom-ref'])
|
||||
del component["properties"][i]
|
||||
|
||||
|
||||
def get_version_from_import_script(file_path: str) -> str:
|
||||
"""A rudimentary parse of a shell script file to extract the static value defined for the VERSION variable."""
|
||||
try:
|
||||
|
|
@ -483,6 +596,8 @@ def main() -> None:
|
|||
endor_bom = endorctl.get_sbom_for_branch(git_info.project, git_info.branch)
|
||||
elif target == "project":
|
||||
endor_bom = endorctl.get_sbom_for_project(git_info.project)
|
||||
else:
|
||||
endor_bom = None
|
||||
|
||||
if not endor_bom:
|
||||
logger.error("Empty result for Endor SBOM!")
|
||||
|
|
@ -559,6 +674,9 @@ def main() -> None:
|
|||
meta_bom["components"].sort(key=lambda c: c["bom-ref"])
|
||||
prev_bom["components"].sort(key=lambda c: c["bom-ref"])
|
||||
|
||||
# Check metadata SBOM for completeness
|
||||
check_metadata_sbom(meta_bom)
|
||||
|
||||
# Create SBOM component lookup dicts
|
||||
endor_components = sbom_components_to_dict(endor_bom)
|
||||
prev_components = sbom_components_to_dict(prev_bom)
|
||||
|
|
@ -632,12 +750,22 @@ def main() -> None:
|
|||
"endor": "",
|
||||
"import_script": "",
|
||||
"metadata": "",
|
||||
"priority_version_source": "",
|
||||
}
|
||||
|
||||
component_key = component["bom-ref"].split("@")[0]
|
||||
|
||||
print_banner("Component: " + component_key)
|
||||
|
||||
############## Priority Version Source ###############
|
||||
# Priority version source, if exists
|
||||
priority_version_source = get_component_priority_version_source(component)
|
||||
if priority_version_source:
|
||||
versions["priority_version_source"] = priority_version_source
|
||||
logger.info("PRIORITY VERSION SOURCE: %s: Set priority version source to '%s'",
|
||||
component_key, priority_version_source)
|
||||
del_component_priority_version_source(component)
|
||||
|
||||
################ Endor Labs ################
|
||||
if component_key in endor_components:
|
||||
# Pop component from dict so we are left with only unmatched components
|
||||
|
|
@ -674,23 +802,33 @@ def main() -> None:
|
|||
process_component_special_cases(component_key, component, versions,
|
||||
git_info.repo_root.as_posix())
|
||||
|
||||
# For the standard workflow, we favor the Endor Labs version, followed by import script, followed by hard coded
|
||||
# Log a warning if Endor and import scripts versions do not match
|
||||
if (versions["endor"] and versions["import_script"] and get_semver_from_release_version(
|
||||
versions["endor"]) != get_semver_from_release_version(versions["import_script"])):
|
||||
logger.debug(",".join([
|
||||
"endor:",
|
||||
versions["endor"],
|
||||
str(versions["endor"]),
|
||||
"semver(endor):",
|
||||
get_semver_from_release_version(versions["endor"]),
|
||||
"import_script:",
|
||||
versions["import_script"],
|
||||
str(versions["import_script"]),
|
||||
"semver(import_script):",
|
||||
get_semver_from_release_version(versions["import_script"]),
|
||||
"priority_version_source:",
|
||||
str(versions["priority_version_source"]),
|
||||
]))
|
||||
logger.warning(
|
||||
"VERSION MISMATCH: %s: Endor version %s does not match import script version %s",
|
||||
component_key, versions['endor'], versions['import_script'])
|
||||
f"VERSION MISMATCH: {component_key}: Endor version {versions['endor']} does not match import script version {versions['import_script']}. 'priority_version_source' from metadata: {versions['priority_version_source']}"
|
||||
)
|
||||
|
||||
# For the standard workflow, we favor the pre-set priority version source,
|
||||
# followed by Endor Labs version, followed by import script, followed by hard coded
|
||||
if versions["priority_version_source"] and versions["priority_version_source"] in versions:
|
||||
version = versions[versions["priority_version_source"]]
|
||||
logger.info(
|
||||
f"VERSION: {component_key}: Using priority_version_source '{priority_version_source}' from metadata file."
|
||||
)
|
||||
else:
|
||||
version = versions["endor"] or versions["import_script"] or versions["metadata"]
|
||||
|
||||
############## Assign Version ###############
|
||||
|
|
@ -717,24 +855,25 @@ def main() -> None:
|
|||
location = location.replace("src/third_party/", "")
|
||||
if location in third_party_folders:
|
||||
third_party_folders[location] += 1
|
||||
logger.debug("THIRD_PARTY FOLDER: %s matched folder %s specified in SBOM",
|
||||
component_key, location)
|
||||
logger.debug(
|
||||
f"THIRD_PARTY FOLDER: {component_key} matched folder {location} specified in SBOM"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"THIRD_PARTY FOLDER: %s lists third-party location folder as %s, which does not exist!",
|
||||
component_key, location)
|
||||
f"THIRD_PARTY FOLDER: {component_key} lists third-party location folder as {location}, which does not exist!"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"THIRD_PARTY FOLDER: %s lists a location as '%s'. Ideally, all third-party components are located under 'src/third_party/'.",
|
||||
component_key, location)
|
||||
f"THIRD_PARTY FOLDER: {component_key} lists a location as '{location}'. Ideally, all third-party components are located under 'src/third_party/'."
|
||||
)
|
||||
if not component_defines_location:
|
||||
logger.warning(
|
||||
"THIRD_PARTY FOLDER: %s does not define a location in '.evidence.occurrences[]'",
|
||||
component_key)
|
||||
f"THIRD_PARTY FOLDER: {component_key} does not define a location in '.evidence.occurrences[]'"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"VERSION NOT FOUND: Could not find a version for %s! Removing from SBOM. Component may need to be removed from the %s file.",
|
||||
component_key, sbom_metadata_path)
|
||||
f"VERSION NOT FOUND: Could not find a version for {component_key}! Removing from SBOM. Component may need to be removed from the {sbom_metadata_path} file."
|
||||
)
|
||||
del component
|
||||
|
||||
print_banner("Third Party Folders")
|
||||
|
|
@ -744,7 +883,7 @@ def main() -> None:
|
|||
}
|
||||
if third_party_folders_missed:
|
||||
logger.warning(
|
||||
"THIRD_PARTY FOLDERS: 'src/third_party' folders not matched with a component: %s",
|
||||
"THIRD_PARTY FOLDERS: 'src/third_party' folders not matched with a component: " +
|
||||
",".join(third_party_folders_missed.keys()))
|
||||
else:
|
||||
logger.info(
|
||||
|
|
@ -762,15 +901,15 @@ def main() -> None:
|
|||
print_banner("New Endor Labs components")
|
||||
if endor_components:
|
||||
logger.info(
|
||||
"ENDOR SBOM: There are %s unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM for the next run.",
|
||||
len(endor_components))
|
||||
f"ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM for the next run."
|
||||
)
|
||||
for component in endor_components:
|
||||
# set scope to excluded by default until the component is evaluated
|
||||
endor_components[component]["scope"] = "excluded"
|
||||
meta_bom["components"].append(endor_components[component])
|
||||
meta_bom["dependencies"].append(
|
||||
{"ref": endor_components[component]["bom-ref"], "dependsOn": []})
|
||||
logger.info("SBOM AS-IS COMPONENT: Added %s", component)
|
||||
logger.info(f"SBOM AS-IS COMPONENT: Added {component}")
|
||||
|
||||
# endregion Parse unmatched Endor Labs components
|
||||
|
||||
|
|
@ -779,28 +918,28 @@ def main() -> None:
|
|||
# Have the SBOM app version changed?
|
||||
sbom_app_version_changed = (prev_bom["metadata"]["component"]["version"] !=
|
||||
meta_bom["metadata"]["component"]["version"])
|
||||
logger.info("SUMMARY: MongoDB version changed: %s", sbom_app_version_changed)
|
||||
logger.info(f"SUMMARY: MongoDB version changed: {sbom_app_version_changed}")
|
||||
|
||||
# Have the components changed?
|
||||
prev_components = sbom_components_to_dict(prev_bom, with_version=True)
|
||||
meta_components = sbom_components_to_dict(meta_bom, with_version=True)
|
||||
sbom_components_changed = prev_components.keys() != meta_components.keys()
|
||||
logger.info(
|
||||
"SBOM_DIFF: SBOM components changed (added, removed, or version): %s. Previous SBOM has %s components; New SBOM has %s components",
|
||||
sbom_components_changed, len(prev_components), len(meta_components))
|
||||
f"SBOM_DIFF: SBOM components changed (added, removed, or version): {sbom_components_changed}. Previous SBOM has {len(prev_components)} components; New SBOM has {len(meta_components)} components"
|
||||
)
|
||||
|
||||
# Components in prev SBOM but not in generated SBOM
|
||||
prev_components = sbom_components_to_dict(prev_bom, with_version=False)
|
||||
meta_components = sbom_components_to_dict(meta_bom, with_version=False)
|
||||
prev_components_diff = list(set(prev_components.keys()) - set(meta_components.keys()))
|
||||
if prev_components_diff:
|
||||
logger.info("SBOM_DIFF: Components in previous SBOM and not in generated SBOM: %s",
|
||||
logger.info("SBOM_DIFF: Components in previous SBOM and not in generated SBOM: " +
|
||||
",".join(prev_components_diff))
|
||||
|
||||
# Components in generated SBOM but not in prev SBOM
|
||||
meta_components_diff = list(set(meta_components.keys()) - set(prev_components.keys()))
|
||||
if meta_components_diff:
|
||||
logger.info("SBOM_DIFF: Components in generated SBOM and not in previous SBOM: %s",
|
||||
logger.info("SBOM_DIFF: Components in generated SBOM and not in previous SBOM: " +
|
||||
",".join(meta_components_diff))
|
||||
|
||||
# serialNumber https://cyclonedx.org/docs/1.5/json/#serialNumber
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@
|
|||
"group": "google.opensource",
|
||||
"name": "Abseil Common Libraries (C++)",
|
||||
"version": "{{VERSION}}",
|
||||
"description": "Abseil is an open-source collection of C++ code (compliant to C++17) designed to augment the C++ standard library.",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -101,6 +102,10 @@
|
|||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/abseil-cpp/scripts/import.sh"
|
||||
},
|
||||
{
|
||||
"name": "generate_sbom:priority_version_source",
|
||||
"value": "import_script"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -122,6 +127,7 @@
|
|||
"group": "arximboldi",
|
||||
"name": "immer",
|
||||
"version": "{{VERSION}}",
|
||||
"description": "Postmodern immutable and persistent data structures for C++ — value semantics at scale",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -163,6 +169,7 @@
|
|||
"group": "chriskohlhoff",
|
||||
"name": "Asio C++ Library",
|
||||
"version": "{{VERSION}}",
|
||||
"description": "Asio is a cross-platform C++ library for network and low-level I/O programming that provides developers with a consistent asynchronous model using a modern C++ approach.",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -210,6 +217,7 @@
|
|||
"group": "google.opensource",
|
||||
"name": "benchmark",
|
||||
"version": "{{VERSION}}",
|
||||
"description": "A microbenchmark support library",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -286,10 +294,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/boost/scripts/import.sh"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -478,6 +482,10 @@
|
|||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/timelib/scripts/import.sh"
|
||||
},
|
||||
{
|
||||
"name": "generate_sbom:priority_version_source",
|
||||
"value": "import_script"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -734,10 +742,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "true"
|
||||
},
|
||||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/snappy/scripts/import.sh"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1099,6 +1103,10 @@
|
|||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/libmongocrypt/import.sh"
|
||||
},
|
||||
{
|
||||
"name": "generate_sbom:priority_version_source",
|
||||
"value": "import_script"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1120,6 +1128,7 @@
|
|||
"group": "confluentinc",
|
||||
"name": "librdkafka - The Apache Kafka C/C++ library",
|
||||
"version": "{{VERSION}}",
|
||||
"description": "The Apache Kafka C/C++ library",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -1148,6 +1157,10 @@
|
|||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/librdkafka/scripts/librdkafka_get_sources.sh"
|
||||
},
|
||||
{
|
||||
"name": "generate_sbom:priority_version_source",
|
||||
"value": "import_script"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1282,6 +1295,12 @@
|
|||
],
|
||||
"copyright": "Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com>. Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>. All rights reserved.",
|
||||
"purl": "pkg:github/antirez/linenoise@6cdc775807e57b2c3fd64bd207814f8ee1fe35f3",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "https://github.com/antirez/linenoise",
|
||||
"type": "distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "internal:team_responsible",
|
||||
|
|
@ -1290,10 +1309,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"name": "info_link",
|
||||
"value": "https://github.com/antirez/linenoise"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1320,6 +1335,7 @@
|
|||
"author": "MongoDB, Inc.",
|
||||
"group": "mongodb",
|
||||
"name": "MongoDB C Driver",
|
||||
"description": "The Official MongoDB driver for C language",
|
||||
"version": "{{VERSION}}",
|
||||
"licenses": [
|
||||
{
|
||||
|
|
@ -1373,6 +1389,7 @@
|
|||
"group": "mozilla",
|
||||
"name": "Mozilla Firefox ESR",
|
||||
"version": "{{VERSION}}",
|
||||
"description": "The C++-only SpiderMonkey component of FireFox ESR used by MongoDB.",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -1429,6 +1446,12 @@
|
|||
],
|
||||
"copyright": "\u00a9 Copyright 2017 Michael Park Distributed under the Boost Software License, Version 1.0",
|
||||
"purl": "pkg:github/mpark/variant@v1.4.0",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "https://github.com/mpark/variant.git",
|
||||
"type": "vcs"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "internal:team_responsible",
|
||||
|
|
@ -1437,10 +1460,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"name": "info_link",
|
||||
"value": "https://github.com/mpark/variant"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1469,6 +1488,12 @@
|
|||
],
|
||||
"copyright": "Copyright (c) 2015-2018 Will Bond <will@wbond.net>",
|
||||
"purl": "pkg:pypi/ocspbuilder@0.10.2",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "https://pypi.org/project/ocspbuilder/",
|
||||
"type": "distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "internal:team_responsible",
|
||||
|
|
@ -1477,10 +1502,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"name": "info_link",
|
||||
"value": "https://github.com/wbond/ocspbuilder"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1515,6 +1536,12 @@
|
|||
],
|
||||
"copyright": "Copyright 2016 Threema GmbH",
|
||||
"purl": "pkg:pypi/ocspresponder@0.5.0",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "https://pypi.org/project/ocspresponder/",
|
||||
"type": "distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "internal:team_responsible",
|
||||
|
|
@ -1523,10 +1550,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"name": "info_link",
|
||||
"value": "https://github.com/threema-ch/ocspresponder"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1611,7 +1634,7 @@
|
|||
}
|
||||
],
|
||||
"copyright": "Copyright 2008 Google Inc. Copyright 2023 Google LLC. All rights reserved.",
|
||||
"cpe": "cpe:2.3:a:google:protobuf:4.25.0:*:*:*:*:*:*:*",
|
||||
"cpe": "cpe:2.3:a:google:protobuf:{{VERSION}}:*:*:*:*:*:*:*",
|
||||
"purl": "pkg:github/protocolbuffers/protobuf@{{VERSION}}",
|
||||
"externalReferences": [
|
||||
{
|
||||
|
|
@ -1703,6 +1726,7 @@
|
|||
"group": "scons",
|
||||
"name": "SCons - a Software Construction tool",
|
||||
"version": "4.9.1",
|
||||
"description": "Open Source next-generation build tool.",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -1710,6 +1734,7 @@
|
|||
}
|
||||
}
|
||||
],
|
||||
"copyright": "Copyright (c) 2001 - 2024 The SCons Foundation",
|
||||
"purl": "pkg:pypi/scons@4.9.1",
|
||||
"externalReferences": [
|
||||
{
|
||||
|
|
@ -1743,6 +1768,7 @@
|
|||
"group": "aappleby",
|
||||
"name": "MurmurHash3",
|
||||
"version": "a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb",
|
||||
"description": "MurmurHash is a non-cryptographic hash function suitable for general hash-based lookup.",
|
||||
"licenses": [
|
||||
{
|
||||
"license": {
|
||||
|
|
@ -1814,10 +1840,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "true"
|
||||
},
|
||||
{
|
||||
"name": "import_script_path",
|
||||
"value": "src/third_party/libstemmer_c/scripts/import.sh"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -1852,6 +1874,12 @@
|
|||
],
|
||||
"copyright": "Copyright \u00a9 1991–2015 Unicode, Inc",
|
||||
"purl": "pkg:generic/unicode-org/unicode@8.0.0?repository_url=https%3A%2F%2Fwww.unicode.org%2FPublic%2F8.0.0%2F",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "https://www.unicode.org/Public/8.0.0/",
|
||||
"type": "distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "internal:team_responsible",
|
||||
|
|
@ -1860,10 +1888,6 @@
|
|||
{
|
||||
"name": "emits_persisted_data",
|
||||
"value": "true"
|
||||
},
|
||||
{
|
||||
"name": "info_link",
|
||||
"value": "http://www.unicode.org/versions/enumeratedversions.html"
|
||||
}
|
||||
],
|
||||
"evidence": {
|
||||
|
|
@ -2084,7 +2108,7 @@
|
|||
"pkg:github/c-ares/c-ares@{{VERSION}}",
|
||||
"pkg:github/chriskohlhoff/asio@{{VERSION}}",
|
||||
"pkg:github/confluentinc/librdkafka@{{VERSION}}",
|
||||
"pkg:github/cyrusimap/cyrus-sasl@cyrus-sasl-2.1.28",
|
||||
"pkg:github/cyrusimap/cyrus-sasl@cyrus-sasl-{{VERSION}}",
|
||||
"pkg:github/dcleblanc/safeint@{{VERSION}}",
|
||||
"pkg:github/derickr/timelib@{{VERSION}}",
|
||||
"pkg:github/facebook/folly@{{VERSION}}",
|
||||
|
|
@ -2103,16 +2127,12 @@
|
|||
"pkg:github/madler/zlib@{{VERSION}}",
|
||||
"pkg:github/mongodb/libmongocrypt@{{VERSION}}",
|
||||
"pkg:github/mongodb/mongo-c-driver@{{VERSION}}",
|
||||
"pkg:github/mpark/variant@v1.4.0",
|
||||
"pkg:github/pcre2project/pcre2@{{VERSION}}",
|
||||
"pkg:github/protocolbuffers/protobuf@{{VERSION}}",
|
||||
"pkg:github/schemastore/schemastore@6847cfc3a17a04a7664474212db50c627e1e3408",
|
||||
"pkg:github/scons/scons@4.9.1",
|
||||
"pkg:github/snowballstem/snowball@{{VERSION}}",
|
||||
"pkg:github/unicode-org/icu@{{VERSION}}",
|
||||
"pkg:github/wiredtiger/wiredtiger@{{VERSION}}",
|
||||
"pkg:pypi/ocspbuilder@0.10.2",
|
||||
"pkg:pypi/ocspresponder@0.5.0"
|
||||
"pkg:github/wiredtiger/wiredtiger@{{VERSION}}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -2164,7 +2184,7 @@
|
|||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"ref": "pkg:github/cyrusimap/cyrus-sasl@cyrus-sasl-2.1.28",
|
||||
"ref": "pkg:github/cyrusimap/cyrus-sasl@cyrus-sasl-{{VERSION}}",
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3,16 +3,22 @@
|
|||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from github import Github, Repository
|
||||
from github.GithubException import GithubException
|
||||
from github.GithubIntegration import GithubIntegration
|
||||
from github import (
|
||||
GithubException,
|
||||
GithubIntegration,
|
||||
Github,
|
||||
Repository,
|
||||
)
|
||||
|
||||
SBOM_FILES = ["sbom.json", "README.third_party.md"]
|
||||
|
||||
# pylint: disable=C0103
|
||||
|
||||
def get_repository(github_owner, github_repo, app_id, _private_key):
|
||||
|
||||
def get_repository(github_owner, github_repo, app_id, _private_key) -> Repository.Repository:
|
||||
"""Gets the mongo github repository."""
|
||||
app = GithubIntegration(int(app_id), _private_key)
|
||||
installation = app.get_repo_installation(github_owner, github_repo)
|
||||
|
|
@ -22,34 +28,34 @@ def get_repository(github_owner, github_repo, app_id, _private_key):
|
|||
return _repo
|
||||
|
||||
|
||||
def create_branch(base_branch, new_branch) -> None:
|
||||
"""Creates a new branch."""
|
||||
try:
|
||||
assert repo is not None
|
||||
def create_branch(base_branch, new_branch, _repo) -> None:
|
||||
"""Create a new branch or get existing branch."""
|
||||
print(f"Attempting to create branch '{new_branch}' with base branch '{base_branch}'.")
|
||||
base_repo_branch = repo.get_branch(base_branch)
|
||||
ref = f"refs/heads/{new_branch}"
|
||||
repo.create_git_ref(ref=ref, sha=base_repo_branch.commit.sha)
|
||||
print("Created branch.")
|
||||
try:
|
||||
base_repo_branch = _repo.get_branch(base_branch)
|
||||
sha = base_repo_branch.commit.sha
|
||||
_repo.create_git_ref(ref=ref, sha=sha)
|
||||
print(f"Created branch '{new_branch}', ref: {ref}, sha: {sha}")
|
||||
except GithubException as ex:
|
||||
if ex.status == 422:
|
||||
print("Branch already exists. Continuing...")
|
||||
print(f"Branch {new_branch} already exists, ref: {ref}")
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def read_text_file(_file_path: str) -> str:
|
||||
def read_text_file(file_path_str: str) -> str:
|
||||
"""Read a text file and return as string."""
|
||||
content = ""
|
||||
try:
|
||||
with open(_file_path, "r", encoding="utf-8") as _file:
|
||||
with open(file_path_str, "r", encoding="utf-8") as _file:
|
||||
content = _file.read()
|
||||
return content
|
||||
except FileNotFoundError:
|
||||
print(f"ERROR: The file '{_file_path}' was not found.")
|
||||
return f"ERROR: The file '{_file_path}' was not found."
|
||||
except Exception as err:
|
||||
print(f"An error occurred: {err}")
|
||||
return f"An error occurred: {err}"
|
||||
print(f"ERROR: The file '{file_path_str}' was not found.")
|
||||
return f"ERROR: The file '{file_path_str}' was not found."
|
||||
except Exception as ex:
|
||||
print(f"An error occurred: {ex}")
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -88,66 +94,87 @@ if __name__ == "__main__":
|
|||
args.private_key[-29:])
|
||||
|
||||
repo = get_repository(args.github_owner, args.github_repo, args.app_id, private_key)
|
||||
assert isinstance(repo, Repository.Repository)
|
||||
assert repo is not None
|
||||
print("Repo: ", repo)
|
||||
print("repo: ", repo)
|
||||
|
||||
PR_NEEDED = False
|
||||
HAS_UPDATE = False
|
||||
|
||||
for file_path in SBOM_FILES:
|
||||
original_file = repo.get_contents(file_path, ref=f"refs/heads/{args.base_branch}")
|
||||
assert not isinstance(original_file, list)
|
||||
print("original_file: ", original_file)
|
||||
original_content = original_file.decoded_content.decode()
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as file:
|
||||
NEW_CONTENT = file.read()
|
||||
new_content = file.read()
|
||||
except FileNotFoundError:
|
||||
print("Error: file '%s' not found.", file_path)
|
||||
NEW_CONTENT = ""
|
||||
# compare strings without whitespace
|
||||
if "".join(NEW_CONTENT.split()) != "".join(original_content.split()):
|
||||
create_branch(args.base_branch, args.new_branch)
|
||||
new_content = original_content
|
||||
|
||||
# Compare content with removed Endor Labs version to avoid triggering a new SBOM on only that change
|
||||
PATTERN = r'{"name":"EndorLabsInc","version":".*"}'
|
||||
REPL = r'{"name":"EndorLabsInc","version":""}'
|
||||
original_content_compare = re.sub(PATTERN, REPL, "".join(original_content.split()))
|
||||
new_content_compare = re.sub(PATTERN, REPL, "".join(new_content.split()))
|
||||
|
||||
if original_content_compare != new_content_compare:
|
||||
create_branch(args.base_branch, args.new_branch, repo)
|
||||
original_file_new_branch = repo.get_contents(file_path,
|
||||
ref=f"refs/heads/{args.new_branch}")
|
||||
print("original_file_new_branch: ", original_file_new_branch)
|
||||
|
||||
print("New file is different from original file.")
|
||||
print("repo.update_file:")
|
||||
print(f" message: Updating '{file_path}'")
|
||||
print(f" path: '{file_path}'")
|
||||
print(" path: ", file_path)
|
||||
assert not isinstance(original_file_new_branch, list)
|
||||
print(f" sha: {original_file_new_branch.sha}")
|
||||
print(f" content: '{NEW_CONTENT:.256}'")
|
||||
print(f" branch: {args.new_branch}")
|
||||
print(" sha: ", original_file_new_branch.sha)
|
||||
print(" content:")
|
||||
print(new_content[:128])
|
||||
print("...[truncated]...")
|
||||
print(new_content[-128:])
|
||||
print(" branch: ", args.new_branch)
|
||||
time.sleep(10) # Wait to reduce chance of 409 errors
|
||||
update_file_result = repo.update_file(
|
||||
message=f"Updating '{file_path}'",
|
||||
path=file_path,
|
||||
sha=original_file_new_branch.sha,
|
||||
content=NEW_CONTENT,
|
||||
content=new_content,
|
||||
branch=args.new_branch,
|
||||
)
|
||||
print("Results:")
|
||||
print(" commit: ", update_file_result["commit"])
|
||||
print("update_file_result: ", update_file_result)
|
||||
commit = update_file_result.get("commit")
|
||||
print("commit: ", commit)
|
||||
|
||||
PR_NEEDED = True
|
||||
|
||||
if PR_NEEDED:
|
||||
PR_BODY = "Automated PR updating SBOM and related files."
|
||||
if args.saved_warnings:
|
||||
PR_BODY += "\n\nThe following warnings were output by the SBOM generation script:\n"
|
||||
PR_BODY += read_text_file(args.saved_warnings)
|
||||
HAS_UPDATE = True
|
||||
|
||||
if HAS_UPDATE:
|
||||
# Get open PR or create new PR
|
||||
pull_requests = repo.get_pulls(state="open", head=f"{args.github_owner}:{args.new_branch}",
|
||||
base=args.base_branch)
|
||||
if pull_requests.totalCount:
|
||||
pull_request = pull_requests[0]
|
||||
print("pull_request: ", pull_request)
|
||||
else:
|
||||
pr_body = "Automated PR updating SBOM and related files."
|
||||
print("Creating PR:")
|
||||
print(f"base={args.base_branch}")
|
||||
print(f"head={args.new_branch}")
|
||||
print(f"title={args.pr_title}")
|
||||
print(f"body={PR_BODY}")
|
||||
print(f" title={args.pr_title}")
|
||||
print(f" head={args.new_branch}")
|
||||
print(f" base={args.base_branch}")
|
||||
print(f" body={pr_body}")
|
||||
|
||||
repo.create_pull(
|
||||
base=args.base_branch,
|
||||
head=args.new_branch,
|
||||
pull_request = repo.create_pull(
|
||||
title=args.pr_title,
|
||||
body=PR_BODY,
|
||||
head=args.new_branch,
|
||||
base=args.base_branch,
|
||||
body=pr_body,
|
||||
)
|
||||
print("pull_request: ", pull_request)
|
||||
|
||||
if args.saved_warnings:
|
||||
pr_comment = "The following warnings were output by the SBOM generation script:\n"
|
||||
if os.path.isfile(args.saved_warnings):
|
||||
pr_comment += read_text_file(args.saved_warnings)
|
||||
comment = pull_request.create_issue_comment(pr_comment)
|
||||
print("Added PR comment: ", comment)
|
||||
else:
|
||||
print(f"Files '{SBOM_FILES}' have not changed. Skipping PR.")
|
||||
|
|
|
|||
|
|
@ -1,12 +1,20 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for buildscripts/sbom/*.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.append(".")
|
||||
sys.path.append("buildscripts/sbom")
|
||||
|
||||
# pylint: disable=C0413 # wrong-import-position
|
||||
from buildscripts.sbom.config import get_semver_from_release_version, is_valid_purl, regex_semver
|
||||
from buildscripts.sbom.config import get_semver_from_release_version, regex_semver
|
||||
from buildscripts.sbom.endorctl_utils import EndorCtl
|
||||
from buildscripts.sbom.generate_sbom import is_valid_purl
|
||||
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
||||
|
||||
|
|
@ -171,5 +179,42 @@ class TestConfigRegex(unittest.TestCase):
|
|||
self.assertFalse(is_valid_purl(purl), f"Expected '{purl}' to be invalid")
|
||||
|
||||
|
||||
class TestMetadataFile(unittest.TestCase):
|
||||
TEST_DIR = os.path.join("buildscripts", "sbom")
|
||||
VERSION_TAG = "{{VERSION}}"
|
||||
|
||||
def read_sbom_json_file(self, file_path: str) -> dict:
|
||||
"""Load a JSON SBOM file (schema is not validated)."""
|
||||
with open(file_path, "r", encoding="utf-8") as input_json:
|
||||
sbom_json = input_json.read()
|
||||
return json.loads(sbom_json)
|
||||
|
||||
def test_metadata_sbom_version_tags(self):
|
||||
sbom_metadata_file = os.path.join(self.TEST_DIR, "metadata.cdx.json")
|
||||
print(sbom_metadata_file)
|
||||
meta_bom = self.read_sbom_json_file(sbom_metadata_file)
|
||||
for component in meta_bom["components"]:
|
||||
with self.subTest(component=component):
|
||||
properties = []
|
||||
properties.append(component["bom-ref"])
|
||||
properties.append(component["version"])
|
||||
if "purl" in component:
|
||||
properties.append(component["purl"])
|
||||
if "cpe" in component:
|
||||
properties.append(component["cpe"])
|
||||
# make sure component has a minimum of bom-ref, version and at least one of purl or cpe
|
||||
self.assertGreater(
|
||||
len(properties),
|
||||
2,
|
||||
f"Component must have a minimum of bom-ref, version and at least one of purl or cpe. {properties}",
|
||||
)
|
||||
# make sure all properites either have version tag or no version tags
|
||||
self.assertTrue(
|
||||
all(self.VERSION_TAG in p for p in properties)
|
||||
or all(self.VERSION_TAG not in p for p in properties),
|
||||
f"Component must have version tag '{self.VERSION_TAG}' in all or none of bom-ref, version and purl and/or cpe. {properties})",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ tasks:
|
|||
- "--endorctl-path=${workdir}/endorctl"
|
||||
- "--config-path=${workdir}/.endorctl"
|
||||
- "--namespace=${ENDOR_NAMESPACE}"
|
||||
- "--save-warnings=${workdir}/warnings-generate_sbom.txt"
|
||||
- "--save-warnings=${workdir}/generate_sbom_py_warnings.txt"
|
||||
- command: subprocess.exec
|
||||
display_name: Generate third-party readme file
|
||||
params:
|
||||
|
|
@ -131,9 +131,9 @@ tasks:
|
|||
- "--github-owner=${github_org}"
|
||||
- "--github-repo=${github_repo}"
|
||||
- "--base-branch=${branch_name}"
|
||||
- "--new-branch=SERVER-111072/sbom_update_${revision}"
|
||||
- "--pr-title=SERVER-111072_Auto-generated_SBOM_files_[${branch_name}]_${revision}"
|
||||
- "--saved-warnings=${workdir}/warnings-generate_sbom.txt"
|
||||
- "--new-branch=SERVER-111072/sbom_update_${branch_name}"
|
||||
- "--pr-title=SERVER-111072_Auto-generated_SBOM_files_[${branch_name}]"
|
||||
- "--saved-warnings=${workdir}/generate_sbom_py_warnings.txt"
|
||||
|
||||
- name: upload_sbom_via_silkbomb_if_changed
|
||||
allowed_requesters: ["commit", "patch"]
|
||||
|
|
|
|||
|
|
@ -16,6 +16,22 @@ buildvariants:
|
|||
- devprod_coverity
|
||||
tasks:
|
||||
- name: publish-sast-report
|
||||
|
||||
- name: test-release-sbom
|
||||
activate: false
|
||||
display_name: "Test Release SBOM Upload"
|
||||
# The build variant is aimed for publish-augmented-sbom task testing.
|
||||
# For end-to-end tests:
|
||||
## - Create new test branch from 10gen/mongo
|
||||
## - Change SilkBomb branch argument to the new test branch in the task.
|
||||
## - Cleanup results in DependencyTracker.
|
||||
# Note: The task may use "Admin Only" variables, so patch runs may only succeed for Evergreen Project admins
|
||||
allowed_requesters: ["patch"]
|
||||
tags: ["assigned_to_jira_team_platsec_server"]
|
||||
run_on: ubuntu2404-small
|
||||
modules:
|
||||
- devprod_coverity
|
||||
tasks:
|
||||
- name: publish-augmented-sbom
|
||||
|
||||
- name: upload-sbom-if-changed
|
||||
|
|
@ -35,8 +51,8 @@ buildvariants:
|
|||
display_name: "Generate SBOM files and create PR"
|
||||
# Don't run as part of patch builds
|
||||
patchable: false
|
||||
# Run at 6 am UTC daily
|
||||
cron: "0 6 * * *"
|
||||
# Run at 6 am UTC Mon-Fri
|
||||
cron: "0 6 * * 1-5"
|
||||
run_on: rhel92-small
|
||||
expansions:
|
||||
ENDOR_NAMESPACE: mongodb.10gen
|
||||
|
|
|
|||
Loading…
Reference in New Issue