mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-107852 SBOM rebuild to v8.1 (#39748) 

GitOrigin-RevId: 408d4a18c1a37ca9f4e05c7521d641eca4fcbec7
This commit is contained in:
Jason Hills 2025-09-08 10:00:23 -04:00 committed by MongoDB Bot
parent 5c77c39715
commit f4ead726fd
25 changed files with 1824 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.prettierignore
View File

@ -36,6 +36,9 @@ version_expansions.yml
# Ignore all formatting in third_party/* # Ignore all formatting in third_party/*
src/third_party src/third_party
# this file is automatically generated and conforms to formatting requirements
README.third_party.md
# Ignore anything in the build output directories # Ignore anything in the build output directories
build build
bazel-* bazel-*

211
README.third_party.md
View File

@ -21,131 +21,140 @@ not authored by MongoDB, and has a license which requires reproduction,
a notice will be included in a notice will be included in
`THIRD-PARTY-NOTICES`. `THIRD-PARTY-NOTICES`.
| Name | License | Vendored Version | Emits persisted data | Distributed in Release Binaries | | Name | License | Vendored Version | Emits persisted data | Distributed in Release Binaries |
| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ---------------------------------------- | -------------------- | ------------------------------- | | ---------------------------------------------------- | --------------------------------- | ---------------------------------------- | -------------------- | ------------------------------- |
| [Abseil] | Apache-2.0 | 20230802.1 | | ✗ | | [Abseil Common Libraries (C++)] | Apache-2.0 | 20230802.1 | | ✗ |
| [arximboldi/immer] | BSL-1.0 | Unknown | | ✗ | | [Asio C++ Library] | BSL-1.0 | 1.12.2 | | ✗ |
| [Asio C++ Library] | BSL-1.0 | 1.12.2 | | ✗ | | [AWS SDK for C++] | Apache-2.0 | 1.11.471 | | ✗ |
| [aws-sdk - the AWS SDK client library] | Apache-2.0 | 1.11.471 | | ✗ | | [benchmark] | Apache-2.0 | v1.5.2 | | |
| [benchmark] | Apache-2.0 | v1.5.2 | | | | [Boost C++ Libraries] | BSL-1.0 | 1.79.0 | | ✗ |
| [Boost C++ Libraries - boost] | BSL-1.0 | 1.79.0 | | ✗ | | [c-ares] | MIT | 1.27.0 | | ✗ |
| [c-ares] | MIT | 1.27.0 | | ✗ | | [CRoaring] | Apache-2.0 OR MIT | 3.0.1 | | ✗ |
| [concurrencytest] | GPL-3.0-or-later | 0.1.2 | unknown | | | [Cyrus SASL] | BSD-Attribution-HPND-disclaimer | 2.1.28 | | |
| [Cyrus SASL] | BSD-Attribution-HPND-disclaimer | 2.1.28 | unknown | | | [fmt] | MIT | 11.1.3 | | ✗ |
| [dcleblanc/SafeInt] | MIT | 3.0.26 | | ✗ | | [folly] | Apache-2.0 | v2025.04.21.00 | | ✗ |
| [derickr/timelib] | MIT | 2022.13 | | ✗ | | [gperftools] | BSD-3-Clause | 2.9.1 | | ✗ |
| [discover] | BSD-3-Clause | 0.4.0 | unknown | | | [gRPC (C++)] | Apache-2.0 | 1.59.5 | | ✗ |
| [fmtlib/fmt] | MIT | 11.1.3 | | ✗ | | [immer] | BSL-1.0 | 0.8.0 | | ✗ |
| [google-re2] | BSD-3-Clause | 2023-11-01 | | ✗ | | [Intel® Decimal Floating-Point Math Library] | BSD-3-Clause | v2.0U1 | | ✗ |
| [google-snappy] | BSD-3-Clause | 1.1.10 | ✗ | ✗ | | [International Components for Unicode C/C++ (ICU4C)] | Unicode-3.0 | 57.1 | ✗ | ✗ |
| [google/s2geometry] | Apache-2.0 | Unknown | ✗ | ✗ | | [JSON Schema Store] | Apache-2.0 | 6847cfc3a17a04a7664474212db50c627e1e3408 | | |
| [gperftools] | BSD-3-Clause | 2.9.1 | | ✗ | | [JSON-Schema-Test-Suite] | MIT | 728066f9c5c258ba3b1804a22a5b998f2ec77ec0 | | |
| [grpc] | Apache-2.0 | 1.59.5 | | ✗ | | [libmongocrypt] | Apache-2.0 | 1.12.0 | ✗ | ✗ |
| [ICU for C/C++ (ICU4C)] | BSD-3-Clause, MIT v2 with Ad Clause License, Public Domain, BSD-2-Clause | 57.1 | ✗ | ✗ | | [librdkafka - The Apache Kafka C/C++ library] | BSD-2-Clause | 2.0.2 | | ✗ |
| [Intel Decimal Floating-Point Math Library] | BSD-3-Clause | v2.0 U1 | | ✗ | | [LibTomCrypt] | Unlicense | 1.18.2 | ✗ | ✗ |
| [jbeder/yaml-cpp] | MIT | 0.6.3 | | ✗ | | [libunwind] | MIT | v1.8.1 | | ✗ |
| [JSON-Schema-Test-Suite] | Unknown License | Unknown | | | | [linenoise] | BSD-2-Clause | 6cdc775807e57b2c3fd64bd207814f8ee1fe35f3 | | ✗ |
| [libmongocrypt] | Apache-2.0 | 1.12.0 | ✗ | ✗ | | [MongoDB C Driver] | Apache-2.0 | 1.27.6 | ✗ | ✗ |
| [librdkafka - the Apache Kafka C/C++ client library] | BSD-3-Clause, Xmlproc License, ISC, MIT, Public Domain, Zlib, BSD-2-Clause, Andreas Stolcke License | 2.0.2 | | ✗ | | [Mozilla Firefox ESR] | MPL-2.0 | 128.11.0esr | | ✗ |
| [LibTomCrypt] | WTFPL, Public Domain | 1.18.2 | ✗ | ✗ | | [MurmurHash3] | Public Domain | a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb | | ✗ |
| [libunwind/libunwind] | MIT | v1.8.1 | | ✗ | | [nlohmann/json] | MIT | 3.10.5 | | |
| [linenoise] | BSD-2-Clause | Unknown | | ✗ | | [nlohmann/json] | MIT | 3.11.3 | ✗ | |
| [MongoDB C Driver] | Apache-2.0 | 1.27.6 | ✗ | ✗ | | [node] | ISC | 22.1.0 | | |
| [Mozilla Firefox] | MPL-2.0 | 128.11.0esr | unknown | ✗ | | [opentelemetry-cpp] | Apache-2.0 | 1.17 | ✗ | |
| [nlohmann.json.decomposed] | MIT | 3.10.5 | unknown | | | [opentelemetry-proto] | Apache-2.0 | 1.3.2 | ✗ | |
| [node] | ISC | 22.1.0 | unknown | | | [PCRE2 - Perl-Compatible Regular Expressions] | BSD-3-Clause WITH PCRE2-exception | 10.40 | | ✗ |
| [ocspbuilder] | MIT | 0.10.2 | | | | [Protobuf] | BSD-3-Clause | v4.25.0 | | ✗ |
| [ocspresponder] | Apache-2.0 | 0.5.0 | | | | [pypi/asn1crypto] | MIT | 1.5.1 | | |
| [opentelemetry-cpp] | Apache-2.0 | 1.17 | ✗ | | | [pypi/concurrencytest] | GPL-3.0-or-later | 0.1.2 | | |
| [opentelemetry-proto] | Apache-2.0 | 1.3.2 | ✗ | | | [pypi/discover] | BSD-3-Clause | 0.4.0 | | |
| [PCRE2] | BSD-3-Clause, Public Domain | 10.40 | | ✗ | | [pypi/extras] | MIT | 0.0.3 | | |
| [Protobuf] | BSD-3-Clause | v4.25.0 | | ✗ | | [pypi/iso8601] | MIT | 2.1.0 | | |
| [pyiso8601] | MIT | 2.1.0 | unknown | | | [pypi/ocspbuilder] | MIT | 0.10.2 | | |
| [RoaringBitmap/CRoaring] | Unknown License | v3.0.1 | | ✗ | | [pypi/ocspresponder] | Apache-2.0 | 0.5.0 | | |
| [SchemaStore/schemastore] | Apache-2.0 | Unknown | | | | [pypi/oscrypto] | MIT | 1.3.0 | | |
| [SCons - a Software Construction tool] | MIT | 3.1.2 | | ✗ | | [pypi/python-subunit] | (Apache-2.0 OR BSD-3-Clause) | 1.4.4 | | |
| [smhasher] | Unknown License | Unknown | unknown | ✗ | | [pypi/testscenarios] | BSD-3-Clause | 0.4 | | |
| [Snowball Stemming Algorithms] | BSD-3-Clause | 7b264ffa0f767c579d052fd8142558dc8264d795 | ✗ | ✗ | | [pypi/testtools] | MIT | 2.7.1 | | |
| [subunit] | BSD-3-Clause, Apache-2.0 | 1.4.4 | unknown | | | [re2] | BSD-3-Clause | 2023-11-01 | | ✗ |
| [tcmalloc] | Apache-2.0 | 20230227-snapshot-093ba93c | | ✗ | | [S2 Geometry Library] | Apache-2.0 | c872048da5d1 | ✗ | ✗ |
| [testing-cabal/extras] | MIT | 0.0.3 | unknown | | | [SafeInt] | MIT | 3.0.26 | | ✗ |
| [testscenarios] | BSD-3-Clause, Apache-2.0 | 0.4 | unknown | | | [SCons - a Software Construction tool] | MIT | 3.1.2 | | |
| [testtools] | MIT | 2.7.1 | unknown | | | [snappy] | BSD-3-Clause | 1.1.10 | ✗ | ✗ |
| [unicode-data] | Unicode-DFS-2016 | 8.0 | ✗ | ✗ | | [Snowball Stemming Algorithms (libstemmer)] | BSD-3-Clause | 7b264ffa0f767c579d052fd8142558dc8264d795 | ✗ | ✗ |
| [valgrind] | GPL-2.0-or-later | Unknown | | ✗ | | [tcmalloc] | Apache-2.0 | 093ba93c1bd6dca03b0a8334f06d01b019244291 | | ✗ |
| [zlib] | Zlib | v1.3.1 | ✗ | ✗ | | [timelib] | MIT | 2022.13 | | ✗ |
| [zstd] | BSD-3-Clause, GPL-2.0-or-later | 1.5.5 | ✗ | ✗ | | [Unicode Character Database] | Unicode-DFS-2016 | 8.0.0 | ✗ | ✗ |
| [valgrind.h] | BSD-4-Clause | 3.17.0 | | ✗ |
| [WiredTiger] | GPL-2.0-only OR GPL-3.0-only | mongodb-8.1 | ✗ | ✗ |
| [yaml-cpp] | MIT | 0.6.3 | | ✗ |
| [zlib] | Zlib | 1.3.1 | ✗ | ✗ |
| [Zstandard (zstd)] | BSD-3-Clause OR GPL-2.0-only | 1.5.5 | ✗ | ✗ |
[Abseil]: https://github.com/abseil/abseil-cpp [AWS SDK for C++]: https://github.com/aws/aws-sdk-cpp
[Abseil Common Libraries (C++)]: https://github.com/abseil/abseil-cpp
[Asio C++ Library]: https://github.com/chriskohlhoff/asio [Asio C++ Library]: https://github.com/chriskohlhoff/asio
[Boost C++ Libraries - boost]: http://www.boost.org/ [Boost C++ Libraries]: http://www.boost.org/
[CRoaring]: https://github.com/RoaringBitmap/CRoaring
[Cyrus SASL]: https://www.cyrusimap.org/sasl/ [Cyrus SASL]: https://www.cyrusimap.org/sasl/
[ICU for C/C++ (ICU4C)]: http://site.icu-project.org/download/ [Intel® Decimal Floating-Point Math Library]: https://software.intel.com/en-us/articles/intel-decimal-floating-point-math-library
[Intel Decimal Floating-Point Math Library]: https://software.intel.com/en-us/articles/intel-decimal-floating-point-math-library [International Components for Unicode C/C++ (ICU4C)]: http://site.icu-project.org/download/
[JSON Schema Store]: https://www.schemastore.org/json/
[JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite [JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite
[LibTomCrypt]: https://github.com/libtom/libtomcrypt/releases [LibTomCrypt]: https://github.com/libtom/libtomcrypt/releases
[MongoDB C Driver]: https://github.com/mongodb/mongo-c-driver [MongoDB C Driver]: https://github.com/mongodb/mongo-c-driver
[Mozilla Firefox]: https://www.mozilla.org/en-US/security/known-vulnerabilities/firefox-esr [Mozilla Firefox ESR]: https://www.mozilla.org/en-US/security/known-vulnerabilities/firefox-esr
[PCRE2]: http://www.pcre.org/ [MurmurHash3]: https://github.com/aappleby/smhasher/blob/a6bd3ce/
[PCRE2 - Perl-Compatible Regular Expressions]: http://www.pcre.org/
[Protobuf]: https://github.com/protocolbuffers/protobuf [Protobuf]: https://github.com/protocolbuffers/protobuf
[RoaringBitmap/CRoaring]: https://github.com/RoaringBitmap/CRoaring [S2 Geometry Library]: https://github.com/google/s2geometry
[SCons - a Software Construction tool]: https://github.com/SCons/scons [SCons - a Software Construction tool]: https://github.com/SCons/scons
[SchemaStore/schemastore]: https://www.schemastore.org/json/ [SafeInt]: https://github.com/dcleblanc/SafeInt
[Snowball Stemming Algorithms]: https://github.com/snowballstem/snowball [Snowball Stemming Algorithms (libstemmer)]: https://github.com/snowballstem/snowball
[arximboldi/immer]: https://github.com/arximboldi/immer [Unicode Character Database]: http://www.unicode.org/versions/enumeratedversions.html
[aws-sdk - the AWS SDK client library]: https://github.com/aws/aws-sdk-cpp [WiredTiger]: https://source.wiredtiger.com/
[Zstandard (zstd)]: https://github.com/facebook/zstd
[benchmark]: https://github.com/google/benchmark [benchmark]: https://github.com/google/benchmark
[c-ares]: https://c-ares.org/ [c-ares]: https://c-ares.org/
[concurrencytest]: https://pypi.org/project/concurrencytest/ [fmt]: http://fmtlib.net/
[dcleblanc/SafeInt]: https://github.com/dcleblanc/SafeInt [folly]: https://github.com/facebook/folly
[derickr/timelib]: https://github.com/derickr/timelib [gRPC (C++)]: https://github.com/grpc/grpc
[discover]: https://pypi.org/project/discover/
[fmtlib/fmt]: http://fmtlib.net/
[google-re2]: https://github.com/google/re2
[google-snappy]: https://github.com/google/snappy/releases
[google/s2geometry]: https://github.com/google/s2geometry
[gperftools]: https://github.com/gperftools/gperftools [gperftools]: https://github.com/gperftools/gperftools
[grpc]: https://github.com/grpc/grpc [immer]: https://github.com/arximboldi/immer
[jbeder/yaml-cpp]: https://github.com/jbeder/yaml-cpp/releases
[libmongocrypt]: https://github.com/mongodb/libmongocrypt [libmongocrypt]: https://github.com/mongodb/libmongocrypt
[librdkafka - the Apache Kafka C/C++ client library]: https://github.com/confluentinc/librdkafka [librdkafka - The Apache Kafka C/C++ library]: https://github.com/confluentinc/librdkafka
[libunwind/libunwind]: http://www.github.com/libunwind/libunwind [libunwind]: http://www.github.com/libunwind/libunwind
[linenoise]: https://github.com/antirez/linenoise [linenoise]: https://github.com/antirez/linenoise
[nlohmann-json]: https://github.com/open-telemetry/opentelemetry-proto [nlohmann/json]: https://github.com/nlohmann/json
[nlohmann.json.decomposed]: https://www.nuget.org/packages/nlohmann.json.decomposed [nlohmann/json]: https://github.com/open-telemetry/opentelemetry-proto
[node]: https://nodejs.org/en/blog/release [node]: https://nodejs.org/en/blog/release
[ocspbuilder]: https://github.com/wbond/ocspbuilder
[ocspresponder]: https://github.com/threema-ch/ocspresponder
[opentelemetry-cpp]: https://github.com/open-telemetry/opentelemetry-cpp/ [opentelemetry-cpp]: https://github.com/open-telemetry/opentelemetry-cpp/
[opentelemetry-proto]: https://github.com/open-telemetry/opentelemetry-proto [opentelemetry-proto]: https://github.com/open-telemetry/opentelemetry-proto
[pyiso8601]: https://pypi.org/project/iso8601/ [pypi/asn1crypto]: https://github.com/wbond/asn1crypto
[smhasher]: https://github.com/aappleby/smhasher/blob/a6bd3ce/ [pypi/concurrencytest]: https://pypi.org/project/concurrencytest/
[subunit]: https://github.com/testing-cabal/subunit [pypi/discover]: https://pypi.org/project/discover/
[pypi/extras]: https://github.com/testing-cabal/extras
[pypi/iso8601]: https://pypi.org/project/iso8601/
[pypi/ocspbuilder]: https://github.com/wbond/ocspbuilder
[pypi/ocspresponder]: https://github.com/threema-ch/ocspresponder
[pypi/oscrypto]: https://github.com/wbond/oscrypto
[pypi/python-subunit]: https://github.com/testing-cabal/subunit
[pypi/testscenarios]: https://pypi.org/project/testscenarios/
[pypi/testtools]: https://github.com/testing-cabal/testtools
[re2]: https://github.com/google/re2
[snappy]: https://github.com/google/snappy/releases
[tcmalloc]: https://github.com/google/tcmalloc [tcmalloc]: https://github.com/google/tcmalloc
[testing-cabal/extras]: https://github.com/testing-cabal/extras [timelib]: https://github.com/derickr/timelib
[testscenarios]: https://pypi.org/project/testscenarios/ [valgrind.h]: http://valgrind.org/downloads/current.html
[testtools]: https://github.com/testing-cabal/testtools [yaml-cpp]: https://github.com/jbeder/yaml-cpp/releases
[unicode-data]: http://www.unicode.org/versions/enumeratedversions.html
[valgrind]: http://valgrind.org/downloads/current.html
[zlib]: https://zlib.net/ [zlib]: https://zlib.net/
[zstd]: https://github.com/facebook/zstd
## WiredTiger Vendored Test Libraries ## WiredTiger Vendored Test Libraries
The following Python libraries are transitively included by WiredTiger, The following libraries are transitively included by WiredTiger,
and are used by that component for testing. They don't appear in and are used by that component for testing. They don't appear in
released binary artifacts. released binary artifacts.
| Name | | Name |
| ------------------------ | | -------------------------- |
| concurrencytest | | nlohmann/json@3.10.5 |
| discover | | pypi/concurrencytest@0.1.2 |
| nlohmann.json.decomposed | | pypi/discover@0.4.0 |
| pyiso8601 | | pypi/extras@0.0.3 |
| subunit | | pypi/iso8601@2.1.0 |
| testing-cabal/extras | | pypi/python-subunit@1.4.4 |
| testscenarios | | pypi/testscenarios@0.4 |
| testtools | | pypi/testtools@2.7.1 |
## Dynamically Linked Libraries ## Dynamically Linked Libraries

4
buildscripts/BUILD.bazel
View File

@ -256,6 +256,10 @@ py_binary(
"jsonschema", "jsonschema",
group = "build-metrics", group = "build-metrics",
), ),
dependency(
"license-expression",
group = "lint",
),
], ],
) )

96
buildscripts/sbom_linter.py
View File

@ -5,6 +5,7 @@ import sys
from typing import List from typing import List
import jsonschema import jsonschema
from license_expression import get_spdx_licensing
from referencing import Registry, Resource from referencing import Registry, Resource
BOM_SCHEMA_LOCATION = os.path.join("buildscripts", "tests", "sbom_linter", "bom-1.5.schema.json") BOM_SCHEMA_LOCATION = os.path.join("buildscripts", "tests", "sbom_linter", "bom-1.5.schema.json")
@ -32,6 +33,7 @@ MISSING_TEAM_ERROR = "Component must include a 'internal:team_responsible' prope
SCHEMA_MATCH_FAILURE = "File did not match the CycloneDX schema" SCHEMA_MATCH_FAILURE = "File did not match the CycloneDX schema"
MISSING_VERSION_IN_SBOM_COMPONENT_ERROR = "Component must include a version." MISSING_VERSION_IN_SBOM_COMPONENT_ERROR = "Component must include a version."
MISSING_VERSION_IN_IMPORT_FILE_ERROR = "Missing version in the import file: " MISSING_VERSION_IN_IMPORT_FILE_ERROR = "Missing version in the import file: "
MISSING_LICENSE_IN_SBOM_COMPONENT_ERROR = "Component must include a license."
COULD_NOT_FIND_OR_READ_SCRIPT_FILE_ERROR = "Could not find or read the import script file" COULD_NOT_FIND_OR_READ_SCRIPT_FILE_ERROR = "Could not find or read the import script file"
VERSION_MISMATCH_ERROR = "Version mismatch: " VERSION_MISMATCH_ERROR = "Version mismatch: "
@ -115,30 +117,48 @@ def strip_extra_prefixes(string_with_prefix: str) -> str:
return string_with_prefix.removeprefix("mongo/").removeprefix("v") return string_with_prefix.removeprefix("mongo/").removeprefix("v")
def validate_evidence(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None: def validate_license(component: dict, error_manager: ErrorManager) -> None:
if "evidence" not in component or "occurrences" not in component["evidence"]: if "licenses" not in component:
error_manager.append_full_error_message(MISSING_EVIDENCE_ERROR) error_manager.append_full_error_message(MISSING_LICENSE_IN_SBOM_COMPONENT_ERROR)
return return
occurrences = component["evidence"]["occurrences"] valid_license = False
if not occurrences: for license in component["licenses"]:
error_manager.append_full_error_message( if "expression" in license:
"'evidence.occurrences' field must include at least one location." expression = license.get("expression")
) elif "license" in license:
for occurrence in occurrences: if "id" in license["license"]:
location = occurrence["location"] # Should be a valid SPDX license ID
expression = license["license"].get("id")
elif "name" in license["license"]:
# If SPDX does not define the license used, the name field may be used to provide the license name
valid_license = True
if not os.path.exists(location) and not SKIP_FILE_CHECKING: if not valid_license:
error_manager.append_full_error_message("location does not exist in repo.") licensing_validate = get_spdx_licensing().validate(expression, validate=True)
# ExpressionInfo(
# original_expression='',
# normalized_expression='',
# errors=[],
# invalid_symbols=[]
# )
valid_license = not licensing_validate.errors or not licensing_validate.invalid_symbols
if not valid_license:
error_manager.append_full_error_message(licensing_validate)
return
if location.startswith(THIRD_PARTY_LOCATION_PREFIX):
lib = location.removeprefix(THIRD_PARTY_LOCATION_PREFIX) def validate_evidence(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
if lib in third_party_libs: if component["scope"] == "required":
third_party_libs.remove(lib) if "evidence" not in component or "occurrences" not in component["evidence"]:
error_manager.append_full_error_message(MISSING_EVIDENCE_ERROR)
return
validate_location(component, third_party_libs, error_manager)
def validate_properties(component: dict, error_manager: ErrorManager) -> None: def validate_properties(component: dict, error_manager: ErrorManager) -> None:
has_team_responsible_property = False has_team_responsible_property = False or component["scope"] == "excluded"
script_path = "" script_path = ""
if "properties" in component: if "properties" in component:
for prop in component["properties"]: for prop in component["properties"]:
@ -159,14 +179,26 @@ def validate_properties(component: dict, error_manager: ErrorManager) -> None:
if comp_version == "Unknown" or script_path == "": if comp_version == "Unknown" or script_path == "":
return return
# Include the .pedigree.descendants[0] version for version matching
if (
"pedigree" in component
and "descendants" in component["pedigree"]
and "version" in component["pedigree"]["descendants"][0]
):
comp_pedigree_version = component["pedigree"]["descendants"][0]["version"]
else:
comp_pedigree_version = ""
# At this point a version is attempted to be read from the import script file # At this point a version is attempted to be read from the import script file
script_version = get_script_version(script_path, "VERSION", error_manager) script_version = get_script_version(script_path, "VERSION", error_manager)
if script_version == "": if script_version == "":
error_manager.append_full_error_message(MISSING_VERSION_IN_IMPORT_FILE_ERROR + script_path) error_manager.append_full_error_message(MISSING_VERSION_IN_IMPORT_FILE_ERROR + script_path)
elif strip_extra_prefixes(script_version) != strip_extra_prefixes(comp_version): elif strip_extra_prefixes(script_version) != strip_extra_prefixes(
comp_version
) and strip_extra_prefixes(script_version) != strip_extra_prefixes(comp_pedigree_version):
error_manager.append_full_error_message( error_manager.append_full_error_message(
VERSION_MISMATCH_ERROR VERSION_MISMATCH_ERROR
+ f"\nscript version:{script_version}\nsbom version:{comp_version}" + f"\nscript version:{script_version}\nsbom component version:{comp_version}\nsbom component pedigree version:{comp_pedigree_version}"
) )
@ -174,15 +206,37 @@ def validate_component(component: dict, third_party_libs: set, error_manager: Er
error_manager.update_component_attribute(component["name"]) error_manager.update_component_attribute(component["name"])
if "scope" not in component: if "scope" not in component:
error_manager.append_full_error_message("component must include a scope.") error_manager.append_full_error_message("component must include a scope.")
elif component["scope"] != "optional": else:
validate_evidence(component, third_party_libs, error_manager) validate_evidence(component, third_party_libs, error_manager)
validate_properties(component, error_manager) validate_properties(component, error_manager)
validate_license(component, error_manager)
if "purl" not in component and "cpe" not in component: if "purl" not in component and "cpe" not in component:
error_manager.append_full_error_message(MISSING_PURL_CPE_ERROR) error_manager.append_full_error_message(MISSING_PURL_CPE_ERROR)
error_manager.update_component_attribute("") error_manager.update_component_attribute("")
def validate_location(component: dict, third_party_libs: set, error_manager: ErrorManager) -> None:
if "evidence" in component:
if "occurrences" not in component["evidence"]:
error_manager.append_full_error_message(
"'evidence.occurrences' field must include at least one location."
)
occurrences = component["evidence"]["occurrences"]
for occurrence in occurrences:
if "location" in occurrence:
location = occurrence["location"]
if not os.path.exists(location) and not SKIP_FILE_CHECKING:
error_manager.append_full_error_message("location does not exist in repo.")
if location.startswith(THIRD_PARTY_LOCATION_PREFIX):
lib = location.removeprefix(THIRD_PARTY_LOCATION_PREFIX)
if lib in third_party_libs:
third_party_libs.remove(lib)
def lint_sbom( def lint_sbom(
input_file: str, output_file: str, third_party_libs: set, should_format: bool input_file: str, output_file: str, third_party_libs: set, should_format: bool
) -> ErrorManager: ) -> ErrorManager:
@ -257,8 +311,6 @@ def main() -> int:
) )
# the only files in this dir that are not third party libs # the only files in this dir that are not third party libs
third_party_libs.remove("scripts") third_party_libs.remove("scripts")
# wiredtiger will not be included in the sbom since it is considered part of the server
third_party_libs.remove("wiredtiger")
# the only files in the sasl dir are BUILD files to setup the sasl library in Windows # the only files in the sasl dir are BUILD files to setup the sasl library in Windows
third_party_libs.remove("sasl") third_party_libs.remove("sasl")
error_manager = lint_sbom(input_file, output_file, third_party_libs, should_format) error_manager = lint_sbom(input_file, output_file, third_party_libs, should_format)

11
buildscripts/tests/sbom_linter/inputs/sbom_component_empty_version.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
@ -14,6 +14,11 @@
"name": "kafka", "name": "kafka",
"version": "", "version": "",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -34,4 +39,4 @@
} }
} }
] ]
} }

11
buildscripts/tests/sbom_linter/inputs/sbom_component_name_missing.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
@ -14,6 +14,11 @@
"version": "v2.0.2", "version": "v2.0.2",
"scope": "required", "scope": "required",
"cpe": "test_cpe", "cpe": "test_cpe",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"properties": [ "properties": [
{ {
"name": "internal:team_responsible", "name": "internal:team_responsible",
@ -33,4 +38,4 @@
} }
} }
] ]
} }

25
buildscripts/tests/sbom_linter/inputs/sbom_invalid_format.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
@ -16,6 +16,11 @@
"name": "kafka", "name": "kafka",
"version": "2.0.2", "version": "2.0.2",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -36,6 +41,13 @@
"name": "protobuf", "name": "protobuf",
"version": "v4.25.0", "version": "v4.25.0",
"scope": "required", "scope": "required",
"licenses": [
{
"license": {
"id": "BSD-3-Clause"
}
}
],
"purl": "test_purl", "purl": "test_purl",
"properties": [ "properties": [
{ {
@ -58,6 +70,13 @@
"name": "unicode", "name": "unicode",
"version": "8.0", "version": "8.0",
"scope": "optional", "scope": "optional",
"licenses": [
{
"license": {
"id": "Unicode-DFS-2016"
}
}
],
"purl": "test_purl", "purl": "test_purl",
"properties": [ "properties": [
{ {
@ -67,4 +86,4 @@
] ]
} }
] ]
} }

43
buildscripts/tests/sbom_linter/inputs/sbom_invalid_license_expression.json Normal file
View File

@ -0,0 +1,43 @@
{
"$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
"properties": [
{
"name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
}
],
"bomFormat": "CycloneDX",
"specVersion": "1.5",
"version": 1,
"components": [
{
"type": "library",
"name": "kafka",
"version": "v2.0.2",
"licenses": [
{
"expression": "xBSD-3-Clause"
}
],
"scope": "required",
"cpe": "test_cpe",
"properties": [
{
"name": "internal:team_responsible",
"value": "server_security"
},
{
"name": "import_script_path",
"value": "buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh"
}
],
"evidence": {
"occurrences": [
{
"location": "src/third_party/librdkafka"
}
]
}
}
]
}

11
buildscripts/tests/sbom_linter/inputs/sbom_missing_evidence.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
@ -14,6 +14,11 @@
"name": "kafka", "name": "kafka",
"scope": "required", "scope": "required",
"cpe": "test_cpe", "cpe": "test_cpe",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"properties": [ "properties": [
{ {
"name": "internal:team_responsible", "name": "internal:team_responsible",
@ -53,4 +58,4 @@
] ]
} }
] ]
} }

74
buildscripts/tests/sbom_linter/inputs/sbom_missing_license.json Normal file
View File

@ -0,0 +1,74 @@
{
"properties": [
{
"name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
}
],
"bomFormat": "CycloneDX",
"specVersion": "1.5",
"version": 1,
"components": [
{
"type": "library",
"name": "kafka",
"version": "v2.0.2",
"scope": "required",
"cpe": "test_cpe",
"properties": [
{
"name": "internal:team_responsible",
"value": "server_security"
},
{
"name": "import_script_path",
"value": "buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh"
}
],
"evidence": {
"occurrences": [
{
"location": "src/third_party/librdkafka"
}
]
}
},
{
"type": "library",
"name": "protobuf",
"version": "v4.25.0",
"scope": "required",
"purl": "test_purl",
"properties": [
{
"name": "internal:team_responsible",
"value": "server_security"
},
{
"name": "import_script_path",
"value": "buildscripts/tests/sbom_linter/inputs/import_script_with_mongo_prefix_version.sh"
}
],
"evidence": {
"occurrences": [
{
"location": "src/third_party/protobuf"
}
]
}
},
{
"type": "library",
"name": "unicode",
"version": "8.0",
"scope": "optional",
"purl": "test_purl",
"properties": [
{
"name": "internal:team_responsible",
"value": "server_security"
}
]
}
]
}

11
buildscripts/tests/sbom_linter/inputs/sbom_missing_purl.json
View File

@ -2,17 +2,22 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
"type": "library", "type": "library",
"name": "kafka", "name": "kafka",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -59,4 +64,4 @@
] ]
} }
] ]
} }

11
buildscripts/tests/sbom_linter/inputs/sbom_missing_team.json
View File

@ -2,17 +2,22 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
"type": "library", "type": "library",
"name": "kafka", "name": "kafka",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"evidence": { "evidence": {
"occurrences": [ "occurrences": [
@ -54,4 +59,4 @@
] ]
} }
] ]
} }

11
buildscripts/tests/sbom_linter/inputs/sbom_missing_version.json
View File

@ -2,17 +2,22 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
"type": "library", "type": "library",
"name": "kafka", "name": "kafka",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -29,4 +34,4 @@
} }
} }
] ]
} }

51
buildscripts/tests/sbom_linter/inputs/sbom_named_license.json Normal file
View File

@ -0,0 +1,51 @@
{
"properties": [
{
"name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
}
],
"bomFormat": "CycloneDX",
"specVersion": "1.5",
"version": 1,
"components": [
{
"type": "library",
"bom-ref": "pkg:github/aappleby/smhasher@a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb",
"supplier": {
"name": "Austin Appleby"
},
"author": "Austin Appleby",
"group": "aappleby",
"name": "MurmurHash3",
"version": "a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb",
"licenses": [
{
"license": {
"name": "Public Domain"
}
}
],
"copyright": "MurmurHash3 was written by Austin Appleby, and is placed in the public domain. The author hereby disclaims copyright to this source code.",
"purl": "pkg:github/aappleby/smhasher@a6bd3ce7be8ad147ea820a7cf6229a975c0c96bb",
"properties": [
{
"name": "internal:team_responsible",
"value": "Storage Execution"
},
{
"name": "info_link",
"value": "https://github.com/aappleby/smhasher/blob/a6bd3ce/"
}
],
"evidence": {
"occurrences": [
{
"location": "src/third_party/murmurhash3"
}
]
},
"scope": "required"
}
]
}

51
buildscripts/tests/sbom_linter/inputs/sbom_pedigree_version_match.json Normal file
View File

@ -0,0 +1,51 @@
{
"properties": [
{
"name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
}
],
"bomFormat": "CycloneDX",
"specVersion": "1.5",
"version": 1,
"components": [
{
"type": "library",
"name": "kafka",
"version": "v2.0.0",
"scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe",
"pedigree": {
"descendants": [
{
"type": "library",
"name": "kafka-fork",
"version": "v2.0.2"
}
]
},
"properties": [
{
"name": "internal:team_responsible",
"value": "server_security"
},
{
"name": "import_script_path",
"value": "buildscripts/tests/sbom_linter/inputs/kafka_valid_import.sh"
}
],
"evidence": {
"occurrences": [
{
"location": "src/third_party/kafka"
}
]
}
}
]
}

11
buildscripts/tests/sbom_linter/inputs/sbom_script_file_missing.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
@ -14,6 +14,11 @@
"name": "kafka", "name": "kafka",
"version": "2.0.2", "version": "2.0.2",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -34,4 +39,4 @@
} }
} }
] ]
} }

11
buildscripts/tests/sbom_linter/inputs/sbom_script_missing_version.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
@ -14,6 +14,11 @@
"name": "kafka", "name": "kafka",
"version": "2.0.2", "version": "2.0.2",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -34,4 +39,4 @@
} }
} }
] ]
} }

11
buildscripts/tests/sbom_linter/inputs/sbom_version_mismatch.json
View File

@ -2,11 +2,11 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
@ -14,6 +14,11 @@
"name": "kafka", "name": "kafka",
"version": "v4.25.0", "version": "v4.25.0",
"scope": "required", "scope": "required",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
{ {
@ -34,4 +39,4 @@
} }
} }
] ]
} }

23
buildscripts/tests/sbom_linter/inputs/valid_sbom.json
View File

@ -2,17 +2,22 @@
"properties": [ "properties": [
{ {
"name": "comment", "name": "comment",
"value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.6/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details." "value": "SBOM for MDB server product; this file should comply with the format specified here: https://cyclonedx.org/docs/1.5/json/#components_items_publisher; This file is still in development; see https://jira.mongodb.org/browse/DEVPROD-2623 for details."
} }
], ],
"bomFormat": "CycloneDX", "bomFormat": "CycloneDX",
"specVersion": "1.6", "specVersion": "1.5",
"version": 1, "version": 1,
"components": [ "components": [
{ {
"type": "library", "type": "library",
"name": "kafka", "name": "kafka",
"version": "v2.0.2", "version": "v2.0.2",
"licenses": [
{
"expression": "BSD-3-Clause"
}
],
"scope": "required", "scope": "required",
"cpe": "test_cpe", "cpe": "test_cpe",
"properties": [ "properties": [
@ -38,6 +43,13 @@
"name": "protobuf", "name": "protobuf",
"version": "v4.25.0", "version": "v4.25.0",
"scope": "required", "scope": "required",
"licenses": [
{
"license": {
"id": "BSD-3-Clause"
}
}
],
"purl": "test_purl", "purl": "test_purl",
"properties": [ "properties": [
{ {
@ -61,6 +73,13 @@
"type": "library", "type": "library",
"name": "unicode", "name": "unicode",
"version": "8.0", "version": "8.0",
"licenses": [
{
"license": {
"id": "Unicode-DFS-2016"
}
}
],
"scope": "optional", "scope": "optional",
"purl": "test_purl", "purl": "test_purl",
"properties": [ "properties": [

31
buildscripts/tests/sbom_linter/test_sbom.py
View File

@ -103,6 +103,14 @@ class TestSbom(unittest.TestCase):
error_manager = sbom_linter.lint_sbom(test_file, test_file, third_party_libs, False) error_manager = sbom_linter.lint_sbom(test_file, test_file, third_party_libs, False)
self.assert_message_in_errors(error_manager, sbom_linter.VERSION_MISMATCH_ERROR) self.assert_message_in_errors(error_manager, sbom_linter.VERSION_MISMATCH_ERROR)
def test_pedigree_version_match(self):
test_file = os.path.join(self.input_dir, "sbom_pedigree_version_match.json")
third_party_libs = {"kafka"}
error_manager = sbom_linter.lint_sbom(test_file, test_file, third_party_libs, False)
if not error_manager.zero_error():
error_manager.print_errors()
self.assertTrue(error_manager.zero_error())
def test_schema_match_failure(self): def test_schema_match_failure(self):
test_file = os.path.join(self.input_dir, "sbom_component_name_missing.json") test_file = os.path.join(self.input_dir, "sbom_component_name_missing.json")
third_party_libs = {"librdkafka"} third_party_libs = {"librdkafka"}
@ -116,3 +124,26 @@ class TestSbom(unittest.TestCase):
self.assert_message_in_errors( self.assert_message_in_errors(
error_manager, sbom_linter.MISSING_VERSION_IN_SBOM_COMPONENT_ERROR error_manager, sbom_linter.MISSING_VERSION_IN_SBOM_COMPONENT_ERROR
) )
def test_missing_license(self):
test_file = os.path.join(self.input_dir, "sbom_missing_license.json")
third_party_libs = {"librdkafka"}
error_manager = sbom_linter.lint_sbom(test_file, test_file, third_party_libs, False)
self.assert_message_in_errors(
error_manager, sbom_linter.MISSING_LICENSE_IN_SBOM_COMPONENT_ERROR
)
def test_invalid_license_expression(self):
test_file = os.path.join(self.input_dir, "sbom_invalid_license_expression.json")
third_party_libs = {"librdkafka"}
error_manager = sbom_linter.lint_sbom(test_file, test_file, third_party_libs, False)
# print(error_manager.errors)
self.assert_message_in_errors(error_manager, "ExpressionInfo")
def test_named_license(self):
test_file = os.path.join(self.input_dir, "sbom_named_license.json")
third_party_libs = {"murmurhash3"}
error_manager = sbom_linter.lint_sbom(test_file, test_file, third_party_libs, False)
if not error_manager.zero_error():
error_manager.print_errors()
self.assertTrue(error_manager.zero_error())

40
poetry.lock generated
View File

@ -96,6 +96,25 @@ files = [
{file = "blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf"}, {file = "blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf"},
] ]
[[package]]
name = "boolean-py"
version = "5.0"
description = "Define boolean algebras, create and parse boolean expressions and create custom boolean DSL."
optional = false
python-versions = "*"
groups = ["lint"]
markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\""
files = [
{file = "boolean_py-5.0-py3-none-any.whl", hash = "sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9"},
{file = "boolean_py-5.0.tar.gz", hash = "sha256:60cbc4bad079753721d32649545505362c754e121570ada4658b852a3a318d95"},
]
[package.extras]
dev = ["build", "twine"]
docs = ["Sphinx (>=3.3.1)", "doc8 (>=0.8.1)", "sphinx-rtd-theme (>=0.5.0)", "sphinxcontrib-apidoc (>=0.3.0)"]
linting = ["black", "isort", "pycodestyle"]
testing = ["pytest (>=6,!=7.0.0)", "pytest-xdist (>=2)"]
[[package]] [[package]]
name = "boto3" name = "boto3"
version = "1.36.18" version = "1.36.18"
@ -1626,6 +1645,25 @@ six = ">=1.7"
Twisted = "*" Twisted = "*"
"zope.interface" = "*" "zope.interface" = "*"
[[package]]
name = "license-expression"
version = "30.4.4"
description = "license-expression is a comprehensive utility library to parse, compare, simplify and normalize license expressions (such as SPDX license expressions) using boolean logic."
optional = false
python-versions = ">=3.9"
groups = ["lint"]
markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\""
files = [
{file = "license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4"},
{file = "license_expression-30.4.4.tar.gz", hash = "sha256:73448f0aacd8d0808895bdc4b2c8e01a8d67646e4188f887375398c761f340fd"},
]
[package.dependencies]
"boolean.py" = ">=4.0"
[package.extras]
dev = ["Sphinx (>=5.0.2)", "doc8 (>=0.11.2)", "pytest (>=7.0.1)", "pytest-xdist (>=2)", "ruff", "sphinx-autobuild", "sphinx-copybutton", "sphinx-reredirects (>=0.1.2)", "sphinx-rtd-dark-mode (>=1.3.0)", "sphinx-rtd-theme (>=1.0.0)", "sphinxcontrib-apidoc (>=0.4.0)", "twine"]
[[package]] [[package]]
name = "linkify-it-py" name = "linkify-it-py"
version = "2.0.3" version = "2.0.3"
@ -5322,4 +5360,4 @@ oldcrypt = []
[metadata] [metadata]
lock-version = "2.1" lock-version = "2.1"
python-versions = ">=3.10,<4.0" python-versions = ">=3.10,<4.0"
content-hash = "8ff6e91d1b0c712296d60e4eae2b53d7e4c3ce2cbc50ae69d546b83c7e111398" content-hash = "3742eca6165d4c8a9463cabfee9c40ad7cada661fa017f6fc13d375ea417640f"

1
pyproject.toml
View File

@ -123,6 +123,7 @@ tqdm = "*"
colorama = "^0.4.6" colorama = "^0.4.6"
evergreen-lint = "^0.1.9" evergreen-lint = "^0.1.9"
ruff = "^0.6.7" ruff = "^0.6.7"
license-expression = "^30.4.4"
[tool.poetry.group.modules_poc.dependencies] [tool.poetry.group.modules_poc.dependencies]
codeowners = { version = "^0.7.0", markers = "platform_machine != 's390x' and platform_machine != 'ppc64le'" } codeowners = { version = "^0.7.0", markers = "platform_machine != 's390x' and platform_machine != 'ppc64le'" }

1571
sbom.json
View File

File diff suppressed because it is too large Load Diff

8
src/third_party/scripts/README.third_party.md.template vendored
View File

@ -19,17 +19,17 @@ not authored by MongoDB, and has a license which requires reproduction,
a notice will be included in a notice will be included in
`THIRD-PARTY-NOTICES`. `THIRD-PARTY-NOTICES`.
{{ component_chart }} $component_chart
{{ component_links }} $component_links
## WiredTiger Vendored Test Libraries ## WiredTiger Vendored Test Libraries
The following Python libraries are transitively included by WiredTiger, The following libraries are transitively included by WiredTiger,
and are used by that component for testing. They don't appear in and are used by that component for testing. They don't appear in
released binary artifacts. released binary artifacts.
{{ wiredtiger_chart }} $wiredtiger_chart
## Dynamically Linked Libraries ## Dynamically Linked Libraries

145
src/third_party/scripts/gen_thirdpartyreadme.py vendored
View File

@ -1,17 +1,19 @@
from jinja2 import Environment, FileSystemLoader
import sys
import os
import json
import bisect import bisect
import json
import logging import logging
from functools import reduce import os
import sys
import warnings
warnings.filterwarnings("ignore", message="\nYou don't have the C version of NameMapper installed")
from Cheetah.Template import Template
SBOM_PATH = "../../../sbom.json" SBOM_PATH = "../../../sbom.json"
TEMPLATE_PATH = "README.third_party.md.template" TEMPLATE_PATH = "README.third_party.md.template"
README_PATH = "../../../README.third_party.md" README_PATH = "../../../README.third_party.md"
logging.basicConfig(level=logging.INFO, logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
format='%(asctime)s - %(levelname)s - %(message)s')
def main(): def main():
@ -31,7 +33,7 @@ def main():
template_data = { template_data = {
"component_chart": component_chart_string, "component_chart": component_chart_string,
"component_links": component_links_string, "component_links": component_links_string,
"wiredtiger_chart": wiredtiger_chart_string "wiredtiger_chart": wiredtiger_chart_string,
} }
create_markdown_with_template(template_data) create_markdown_with_template(template_data)
@ -45,7 +47,7 @@ def test_filepaths() -> None:
def load_sbom() -> dict: def load_sbom() -> dict:
try: try:
with open(SBOM_PATH, 'r') as file: with open(SBOM_PATH, "r") as file:
sbom = json.load(file) sbom = json.load(file)
logging.info("%s JSON data loaded.", SBOM_PATH) logging.info("%s JSON data loaded.", SBOM_PATH)
return sbom return sbom
@ -63,36 +65,47 @@ def sbom_to_component_chart(sbom: dict) -> list[list[str]]:
name = component["name"] name = component["name"]
license_string = [] license_string = []
for lic in component["licenses"]: for lic in component["licenses"]:
for key in ["id", "name"]: if "license" in lic:
if key in lic["license"]: for key in ["id", "name"]:
license_string.append(lic["license"][key]) if key in lic["license"]:
license_string.append(lic["license"][key])
elif "expression" in lic:
license_string.append(lic["expression"])
license_string = ", ".join(license_string) license_string = ", ".join(license_string)
version = component["version"] version = component["version"]
emits_persisted_data = "unknown" if component["scope"] == "excluded":
for prop in component["properties"]: emits_persisted_data = ""
k, v = prop["name"], prop["value"] else:
if k == "emits_persisted_data": emits_persisted_data = "unknown"
emits_persisted_data = ("", "✗")[v == "true"] if "properties" in component:
distributed_in_release_binaries = ( for prop in component["properties"]:
"", "✗")[component["scope"] == "required"] k, v = prop["name"], prop["value"]
if k == "emits_persisted_data":
emits_persisted_data = ("", "✗")[v == "true"]
distributed_in_release_binaries = ("", "✗")[component["scope"] == "required"]
row = [ row = [
item.replace( item.replace("|", "")
"|", for item in [
"") for item in [
f"[{name}]", f"[{name}]",
license_string, license_string,
version, version,
emits_persisted_data, emits_persisted_data,
distributed_in_release_binaries]] distributed_in_release_binaries,
]
]
bisect.insort(component_chart, row, key=lambda c: c[0].lower()) bisect.insort(component_chart, row, key=lambda c: c[0].lower())
component_chart.insert(0, component_chart.insert(
["Name", 0,
"License", [
"Vendored Version", "Name",
"Emits persisted data", "License",
"Distributed in Release Binaries"]) "Vendored Version",
"Emits persisted data",
"Distributed in Release Binaries",
],
)
return component_chart return component_chart
@ -103,9 +116,7 @@ def sbom_to_component_links_string(sbom: dict) -> list[list[str]]:
for component in components: for component in components:
check_component_validity(component) check_component_validity(component)
info_link = get_component_info_link(component) info_link = get_component_info_link(component)
bisect.insort( bisect.insort(link_list, f"[{component['name'].replace('|', '')}]: {info_link}")
link_list,
f"[{component['name'].replace('|','')}]: {info_link}")
return "\n".join(link_list) return "\n".join(link_list)
@ -120,9 +131,9 @@ def sbom_to_wiredtiger_chart(sbom: dict) -> list[list[str]]:
for location in locations: for location in locations:
if location.startswith("src/third_party/wiredtiger/"): if location.startswith("src/third_party/wiredtiger/"):
bisect.insort( bisect.insort(
wiredtiger_chart, [ wiredtiger_chart,
component["name"].replace( ([component["name"].replace("|", "") + "@" + component["version"]]),
"|", "")]) )
return wiredtiger_chart return wiredtiger_chart
@ -130,9 +141,7 @@ def sbom_to_wiredtiger_chart(sbom: dict) -> list[list[str]]:
def check_component_validity(component) -> None: def check_component_validity(component) -> None:
for required_key in ["name", "version", "licenses"]: for required_key in ["name", "version", "licenses"]:
if required_key not in component: if required_key not in component:
logging.error( logging.error("Error: no key %s found in json. Exiting. JSON dump:", required_key)
"Error: no key %s found in json. Exiting. JSON dump:",
required_key)
logging.error(json.dumps(component)) logging.error(json.dumps(component))
sys.exit(1) sys.exit(1)
@ -140,31 +149,28 @@ def check_component_validity(component) -> None:
def get_component_info_link(component) -> str: def get_component_info_link(component) -> str:
name = component["name"] name = component["name"]
links = [] links = []
for prop in component["properties"]: if "properties" in component:
k, v = prop["name"], prop["value"] for prop in component["properties"]:
if k == "info_link": k, v = prop["name"], prop["value"]
links.append(v) if k == "info_link":
if len(links) != 1: links.append(v)
logging.warning( if len(links) != 1:
"Warning: Expected 1 info_link for %s. Got %d:", logging.warning("Warning: Expected 1 info_link for %s. Got %d:", name, len(links))
name, if len(links) > 1:
len(links)) logging.warning(" ".join(links))
if len(links) > 1: logging.warning("Using first link only.")
logging.warning(" ".join(links)) else:
logging.warning("Using first link only.") logging.warning("Falling back to `purl` value: %s", component["purl"])
else: links.append(component["purl"])
logging.warning( return links[0]
"Falling back to `purl` value: %s", else:
component['purl']) return ""
links.append(component["purl"])
return links[0]
def get_component_locations(component) -> list[str]: def get_component_locations(component) -> list[str]:
if "evidence" not in component or "occurrences" not in component["evidence"]: if "evidence" not in component or "occurrences" not in component["evidence"]:
return [] return []
return [occurence["location"] return [occurence["location"] for occurence in component["evidence"]["occurrences"]]
for occurence in component["evidence"]["occurrences"]]
def right_pad_chart_values(chart: list[list[str]]) -> list[list[str]]: def right_pad_chart_values(chart: list[list[str]]) -> list[list[str]]:
@ -186,16 +192,23 @@ def chart_to_string(chart: list[list[str]]) -> str:
return chart return chart
def create_markdown_with_template(data: str) -> None: def create_markdown_with_template(data):
file_loader = FileSystemLoader('.') output = str(
env = Environment(loader=file_loader) Template.compile(
template = env.get_template(TEMPLATE_PATH) file=TEMPLATE_PATH,
output = template.render(data) compilerSettings={
"commentStartToken": "//",
"directiveStartToken": "!!",
"directiveEndToken": "!!",
},
)(namespaces=[data])
)
with open(README_PATH, 'w') as f: with open(README_PATH, "w") as f:
f.write("[DO NOT MODIFY THIS FILE MANUALLY. It is generated by src/third_party/tools/gen_thirdpartyreadme.py]: #\n\n") f.write(
"[DO NOT MODIFY THIS FILE MANUALLY. It is generated by src/third_party/tools/gen_thirdpartyreadme.py]: #\n\n"
)
f.write(output) f.write(output)
f.write("\n")
logging.info("Markdown file created successfully.") logging.info("Markdown file created successfully.")