Refresh

Author: jasonhills-mongodb

CMakeLists.txt

@@ -56,7 +56,6 @@ else()
     message(WARNING "Unknown compiler... recklessly proceeding without a version check")
 endif()
 
-# Also update etc/purls.txt.
 set(BSON_REQUIRED_VERSION 2.1.2)
 set(MONGOC_REQUIRED_VERSION 2.1.2)
 set(MONGOC_DOWNLOAD_VERSION 2.1.2)

etc/sbom/config.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 """generate_sbom.py config. Operational configuration values stored separately from the core code."""
 
-import json
 import logging
 import re
 
@@ -24,7 +23,16 @@
     "pkg:github/",
 ]
 
-for component in endor_components_remove:
+components_remove = [
+    # Endor Labs includes the main component in 'components'. This is not standard, so we remove it.
+    "10gen/mongo",
+    # should be pkg:github/antirez/linenoise - waiting on Endor Labs fix
+    "amokhuginnsson/replxx",
+    # a transitive dependency of s2 that is not necessary to include
+    "sparsehash/sparsehash",
+]
+
+for component in components_remove:
     for prefix in prefixes:
         endor_components_remove.append(prefix + component)
 
@@ -41,56 +49,6 @@
     ["pkg:c/github.com/", "pkg:github/"],
 ]
 
-# ################ PURL Validation ################
-REGEX_STR_PURL_OPTIONAL = (  # Optional Version (any chars except ? @ #)
-    r"(?:@[^?@#]*)?"
-    # Optional Qualifiers (any chars except @ #)
-    r"(?:\?[^@#]*)?"
-    # Optional Subpath (any chars)
-    r"(?:#.*)?$"
-)
-
-REGEX_PURL = {
-    # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md
-    "deb": re.compile(
-        r"^pkg:deb/"  # Scheme and type
-        # Namespace (organization/user), letters must be lowercase
-        r"(debian|ubuntu)+"
-        r"/"
-        r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL  # Name
-    ),
-    # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md
-    "generic": re.compile(
-        r"^pkg:generic/"  # Scheme and type
-        r"([a-zA-Z0-9._-]+/)?"  # Optional namespace segment
-        r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL  # Name (required)
-    ),
-    # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md
-    "github": re.compile(
-        r"^pkg:github/"  # Scheme and type
-        # Namespace (organization/user), letters must be lowercase
-        r"[a-z0-9-]+"
-        r"/"
-        r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL  # Name (repository)
-    ),
-    # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md
-    "pypi": re.compile(
-        r"^pkg:pypi/"  # Scheme and type
-        r"[a-z0-9_-]+"  # Name, letters must be lowercase, dashes, underscore
-        + REGEX_STR_PURL_OPTIONAL
-    ),
-}
-
-
-def is_valid_purl(purl: str) -> bool:
-    """Validate a GitHub or Generic PURL"""
-    for purl_type, regex in REGEX_PURL.items():
-        if regex.match(purl):
-            logger.debug(f"PURL: {purl} matched PURL type '{purl_type}' regex '{regex.pattern}'")
-            return True
-    return False
-
-
 # ################ Version Transformation ################
 
 # In some cases we need to transform the version string to strip out tag-related text
@@ -140,65 +98,8 @@ def get_semver_from_release_version(release_ver: str) -> str:
 # region special component use-case functions
 
 
-def get_version_from_wiredtiger_import_data(file_path: str) -> str:
-    """Get the info in the 'import.data' file saved in the wiredtiger folder"""
-    try:
-        with open(file_path, "r") as input_json:
-            import_data = input_json.read()
-        result = json.loads(import_data)
-    except Exception as e:
-        logger.error(f"Error loading JSON file from {file_path}")
-        logger.error(e)
-        return None
-    return result.get("commit")
-
-
-def get_version_sasl_from_workspace(file_path: str) -> str:
-    """Determine the version that is pulled for Windows Cyrus SASL by searching WORKSPACE.bazel"""
-    # e.g.,
-    #         "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
-    try:
-        with open(file_path, "r") as file:
-            for line in file:
-                if line.strip().startswith(
-                    '"https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-'
-                ):
-                    return line.strip().split("windows_cyrus_sasl-")[1].split(".zip")[0]
-    except Exception as e:
-        logger.warning(f"Unable to load {file_path}")
-        logger.warning(e)
-    else:
-        return None
-
-
 def process_component_special_cases(
     component_key: str, component: dict, versions: dict, repo_root: str
 ) -> None:
-    ## Special case for Cyrus SASL ##
-    if component_key == "pkg:github/cyrusimap/cyrus-sasl":
-        # Cycrus SASL is optionally loaded as a Windows library, when needed. There is no source code for Endor Labs to scan.
-        # The version of Cyrus SASL that is used is defined in the WORKSPACE.bazel file:
-        #         "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip",
-        # Rather than add the complexity of Bazel queries to this script, we just search the text.
-
-        versions["import_script"] = get_version_sasl_from_workspace(repo_root + "/WORKSPACE.bazel")
-        logger.info(
-            f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'WORKSPACE.bazel' file"
-        )
-
-    ## Special case for wiredtiger ##
-    elif component_key == "pkg:github/wiredtiger/wiredtiger":
-        # MongoDB release branches import wiredtiger commits via a bot. These commits will likely not line up with a release or tag.
-        # Endor labs will try to pull the nearest release/tag, but we want the more precise commit hash, which is stored in:
-        # src/third_party/wiredtiget/import.data
-        occurrences = component.get("evidence", {}).get("occurrences", [])
-        if occurrences:
-            location = occurrences[0].get("location")
-            versions["import_script"] = get_version_from_wiredtiger_import_data(
-                f"{repo_root}/{location}/import.data"
-            )
-            logger.info(
-                f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'import.data' file"
-            )
-
+    pass
 # endregion special component use-case functions

etc/sbom/endorctl_utils.py

@@ -437,7 +437,7 @@ def get_sbom_for_branch(self, git_url: str, branch: str) -> dict:
 
             # ScanResult: search for a completed scan
             filter_str = endor_filter.scan_result(
-                None, project_uuid, repository_version_ref, repository_version_sha
+                EndorContextType.MAIN, project_uuid, repository_version_ref, repository_version_sha
             )
             scan_result = self.get_scan_result(filter_str, retry=False)
             project_uuid = scan_result["meta"]["parent_uuid"]

etc/sbom/generate_sbom.py

@@ -23,7 +23,6 @@
     endor_components_remove,
     endor_components_rename,
     get_semver_from_release_version,
-    is_valid_purl,
     process_component_special_cases,
 )
 from endorctl_utils import EndorCtl
@@ -54,7 +53,6 @@ def emit(self, record):
 # Add the handler to the logger
 logger.addHandler(warning_handler)
 
-
 # Get the absolute path of the script file and directory
 script_path = Path(__file__).resolve()
 script_directory = script_path.parent
@@ -66,6 +64,65 @@ def emit(self, record):
 REGEX_RELEASE_BRANCH = r"^v\d\.\d$"
 REGEX_RELEASE_TAG = r"^r\d\.\d.\d(-\w*)?$"
 
+# ################ PURL Validation ################
+REGEX_STR_PURL_OPTIONAL = (  # Optional Version (any chars except ? @ #)
+    r"(?:@[^?@#]*)?"
+    # Optional Qualifiers (any chars except @ #)
+    r"(?:\?[^@#]*)?"
+    # Optional Subpath (any chars)
+    r"(?:#.*)?$"
+)
+
+REGEX_PURL = {
+    # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md
+    "deb": re.compile(
+        r"^pkg:deb/"  # Scheme and type
+        # Namespace (organization/user), letters must be lowercase
+        r"(debian|ubuntu)+"
+        r"/"
+        r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL  # Name
+    ),
+    # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md
+    "generic": re.compile(
+        r"^pkg:generic/"  # Scheme and type
+        r"([a-zA-Z0-9._-]+/)?"  # Optional namespace segment
+        r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL  # Name (required)
+    ),
+    # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md
+    "github": re.compile(
+        r"^pkg:github/"  # Scheme and type
+        # Namespace (organization/user), letters must be lowercase
+        r"[a-z0-9-]+"
+        r"/"
+        r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL  # Name (repository)
+    ),
+    # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md
+    "pypi": re.compile(
+        r"^pkg:pypi/"  # Scheme and type
+        r"[a-z0-9_-]+"  # Name, letters must be lowercase, dashes, underscore
+        + REGEX_STR_PURL_OPTIONAL
+    ),
+}
+
+
+# Metadata SBOM requirements
+METADATA_FIELDS_REQUIRED = [
+    "type",
+    "bom-ref",
+    "group",
+    "name",
+    "version",
+    "description",
+    "licenses",
+    "copyright",
+    "externalReferences",
+    "scope",
+]
+METADATA_FIELDS_ONE_OF = [
+    ["author", "supplier"],
+    ["purl", "cpe"],
+]
+
 # endregion init
 
 
@@ -80,7 +137,11 @@ def __init__(self):
         try:
             self.repo_root = Path(
                 subprocess.run(
-                    "git rev-parse --show-toplevel", shell=True, text=True, capture_output=True
+                    "git rev-parse --show-toplevel",
+                    shell=True,
+                    text=True,
+                    capture_output=True,
+                    check=True,
                 ).stdout.strip()
             )
             self._repo = Repo(self.repo_root)
@@ -170,6 +231,15 @@ def extract_repo_from_git_url(git_url: str) -> dict:
     }
 
 
+def is_valid_purl(purl: str) -> bool:
+    """Validate a GitHub or Generic PURL"""
+    for purl_type, regex in REGEX_PURL.items():
+        if regex.match(purl):
+            logger.debug(f"PURL: {purl} matched PURL type '{purl_type}' regex '{regex.pattern}'")
+            return True
+    return False
+
+
 def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict:
     """Create a dict of SBOM components with a version-less PURL as the key"""
     components = sbom["components"]
@@ -185,6 +255,23 @@ def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict:
     return components_dict
 
 
+def check_metadata_sbom(meta_bom: dict) -> None:
+    for component in meta_bom["components"]:
+        for field in METADATA_FIELDS_REQUIRED:
+            if field not in component:
+                logger.warning(
+                    f"METADATA: '{component['bom-ref'] or component['name']} is missing required field '{field}'."
+                )
+        for fields in METADATA_FIELDS_ONE_OF:
+            found = False
+            for field in fields:
+                found = found or field in component
+            if not found:
+                logger.warning(
+                    f"METADATA: '{component['bom-ref'] or component['name']} is missing one of fields '{fields}'."
+                )
+
+
 def read_sbom_json_file(file_path: str) -> dict:
     """Load a JSON SBOM file (schema is not validated)"""
     try:
@@ -204,8 +291,8 @@ def write_sbom_json_file(sbom_dict: dict, file_path: str) -> None:
     try:
         file_path = os.path.abspath(file_path)
         with open(file_path, "w", encoding="utf-8") as output_json:
-            json.dump(sbom_dict, output_json, indent=2)
-            output_json.write("\n")
+            formatted_sbom = json.dumps(sbom_dict, indent=2) + "\n"
+            output_json.write(formatted_sbom)
     except Exception as e:
         logger.error(f"Error writing SBOM file to {file_path}")
         logger.error(e)
@@ -449,6 +536,8 @@ def main() -> None:
         endor_bom = endorctl.get_sbom_for_branch(git_info.project, git_info.branch)
     elif target == "project":
         endor_bom = endorctl.get_sbom_for_project(git_info.project)
+    else:
+        endor_bom = None
 
     if not endor_bom:
         logger.error("Empty result for Endor SBOM!")
@@ -466,9 +555,6 @@ def main() -> None:
 
     ## remove uneeded components ##
     # [list]endor_components_remove is defined in config.py
-    # Endor Labs includes the main component in 'components'. This is not standard, so we remove it.
-    endor_components_remove.append(f"pkg:github/{git_info.org}/{git_info.repo}")
-    
     # Reverse iterate the SBOM components list to safely modify in situ
     for i in range(len(endor_bom["components"]) - 1, -1, -1):
         component = endor_bom["components"][i]
@@ -529,6 +615,9 @@ def main() -> None:
     meta_bom["components"].sort(key=lambda c: c["bom-ref"])
     prev_bom["components"].sort(key=lambda c: c["bom-ref"])
 
+    # Check metadata SBOM for completeness
+    check_metadata_sbom(meta_bom)
+
     # Create SBOM component lookup dicts
     endor_components = sbom_components_to_dict(endor_bom)
     prev_components = sbom_components_to_dict(prev_bom)
@@ -537,7 +626,7 @@ def main() -> None:
 
     # Attempt to determine the MongoDB Version being scanned
     logger.debug(
-        f"Available MongoDB version options, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom['metadata'].get('component',{}).get('version')}"
+        f"Available MongoDB version options, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom['metadata']['component']['version']}"
     )
     meta_bom_ref = meta_bom["metadata"]["component"]["bom-ref"]