Deduplicate SPDX IDs with hash suffixes

Author: sethmlarson

sbom.py

@@ -25,7 +25,7 @@
 import typing
 import zipfile
 from pathlib import Path
-from typing import Any, NotRequired, TypedDict, cast
+from typing import Any, LiteralString, NotRequired, TypedDict, cast
 from urllib.request import urlopen
 
 
@@ -90,9 +90,19 @@ class CreationInfo(TypedDict):
     licenseListVersion: str
 
 
-def spdx_id(value: str) -> str:
+# Cache of values that we've seen already. We use this
+# to de-duplicate values and their corresponding SPDX ID.
+_SPDX_IDS_TO_VALUES = {}
+
+
+def spdx_id(value: LiteralString) -> str:
     """Encode a value into characters that are valid in an SPDX ID"""
-    return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value)
+    spdx_id = re.sub(r"[^a-zA-Z0-9.\-]+", "-", value)
+    # To avoid collisions we append a hash suffix.
+    suffix = hashlib.sha256(value.encode()).hexdigest()[:8]
+    spdx_id = f"{spdx_id}-{suffix}"
+    assert _SPDX_IDS_TO_VALUES.setdefault(spdx_id, value) == value
+    return spdx_id
 
 
 def calculate_package_verification_codes(sbom: SBOM) -> None:

tests/sbom/sbom-with-pip-removed.json

@@ -13,9 +13,9 @@
   "packages": [],
   "relationships": [
     {
-      "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING",
+      "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING-497fb0c3",
       "relationshipType": "CONTAINS",
-      "spdxElementId": "SPDXRef-PACKAGE-expat"
+      "spdxElementId": "SPDXRef-PACKAGE-expat-83b93528"
     }
   ]
 }

tests/sbom/sbom-with-pip.json

@@ -12,7 +12,7 @@
   "files": [],
   "packages": [
     {
-      "SPDXID": "SPDXRef-PACKAGE-pip",
+      "SPDXID": "SPDXRef-PACKAGE-pip-ced959c1",
       "name": "pip",
       "versionInfo": "24.0",
       "licenseConcluded": "MIT",
@@ -38,19 +38,19 @@
   ],
   "relationships": [
     {
-      "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING",
+      "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING-497fb0c3",
       "relationshipType": "CONTAINS",
-      "spdxElementId": "SPDXRef-PACKAGE-expat"
+      "spdxElementId": "SPDXRef-PACKAGE-expat-83b93528"
     },
     {
-      "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3",
+      "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3-b7a198af",
       "relationshipType": "DEPENDS_ON",
-      "spdxElementId": "SPDXRef-PACKAGE-pip"
+      "spdxElementId": "SPDXRef-PACKAGE-pip-ced959c1"
     },
     {
-      "relatedSpdxElement": "SPDXRef-PACKAGE-pip",
+      "relatedSpdxElement": "SPDXRef-PACKAGE-pip-ced959c1",
       "relationshipType": "DEPENDS_ON",
-      "spdxElementId": "SPDXRef-PACKAGE-cpython"
+      "spdxElementId": "SPDXRef-PACKAGE-cpython-608f998c"
     }
   ]
 }