Modify apache_kafka.py and related tests for migration #972

Reference: #972

Signed-off-by: John M. Horan <johnmhoran@gmail.com>

Author: johnmhoran

vulnerabilities/importers/apache_kafka.py

@@ -7,92 +7,84 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+# temporarily import json to create output to analyze
+import json
 import os
-from unittest import TestCase
 
-from packageurl import PackageURL
-from univers.version_range import VersionRange
+import pytest
 
-from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import Reference
 from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter
-from vulnerabilities.importers.apache_kafka import to_version_ranges
-from vulnerabilities.package_managers import GitHubTagsAPI
-from vulnerabilities.package_managers import Version
-from vulnerabilities.utils import AffectedPackage
+
+# from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.tests import util_tests
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_kafka", "cve-list.html")
-
-
-class TestApacheKafkaImporter(TestCase):
-    def test_to_version_ranges(self):
-        # Check single version
-        assert [
-            VersionRange.from_scheme_version_spec_string("maven", "=3.2.2")
-        ] == to_version_ranges("3.2.2")
-
-        # Check range with lower and upper bounds
-        assert [
-            VersionRange.from_scheme_version_spec_string("maven", ">=3.2.2, <=3.2.3")
-        ] == to_version_ranges("3.2.2 to 3.2.3")
-
-        # Check range with "and later"
-        assert [
-            VersionRange.from_scheme_version_spec_string("maven", ">=3.2.2")
-        ] == to_version_ranges("3.2.2 and later")
-
-        # Check combination of above cases
-        assert [
-            VersionRange.from_scheme_version_spec_string("maven", ">=3.2.2"),
-            VersionRange.from_scheme_version_spec_string("maven", ">=3.2.2, <=3.2.3"),
-            VersionRange.from_scheme_version_spec_string("maven", "==3.2.2"),
-        ] == to_version_ranges("3.2.2 and later, 3.2.2 to 3.2.3, 3.2.2")
-
-    def test_to_advisory(self):
-        data_source = ApacheKafkaImporter(batch_size=1)
-        data_source.version_api = GitHubTagsAPI(
-            cache={"apache/kafka": [Version("2.1.2"), Version("0.10.2.2")]}
-        )
-        expected_advisories = [
-            Advisory(
-                summary="In Apache Kafka versions between 0.11.0.0 and 2.1.0, it is possible to manually\n    craft a Produce request which bypasses transaction/idempotent ACL validation.\n    Only authenticated clients with Write permission on the respective topics are\n    able to exploit this vulnerability. Users should upgrade to 2.1.1 or later\n    where this vulnerability has been fixed.",
-                vulnerability_id="CVE-2018-17196",
-                affected_packages=[
-                    AffectedPackage(
-                        vulnerable_package=PackageURL(
-                            type="apache",
-                            namespace=None,
-                            name="kafka",
-                            version="0.10.2.2",
-                            qualifiers={},
-                            subpath=None,
-                        ),
-                        patched_package=PackageURL(
-                            type="apache",
-                            namespace=None,
-                            name="kafka",
-                            version="2.1.2",
-                            qualifiers={},
-                            subpath=None,
-                        ),
-                    )
-                ],
-                references=[
-                    Reference(
-                        reference_id="", url="https://kafka.apache.org/cve-list", severities=[]
-                    ),
-                    Reference(
-                        reference_id="CVE-2018-17196",
-                        url="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-17196",
-                        severities=[],
-                    ),
-                ],
-            )
-        ]
-        with open(TEST_DATA) as f:
-            found_advisories = data_source.to_advisory(f)
-
-        found_advisories = list(map(Advisory.normalized, found_advisories))
-        expected_advisories = list(map(Advisory.normalized, expected_advisories))
-        assert sorted(found_advisories) == sorted(expected_advisories)
+# Created cve-list-2022-12-06.html with a full copy of https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html
+TEST_DATA = os.path.join(
+    BASE_DIR,
+    "test_data/apache_kafka",
+)
+
+
+def test_to_advisory():
+    with open(os.path.join(TEST_DATA, "cve-list-2022-12-06.html")) as f:
+        raw_data = f.read()
+    advisories = ApacheKafkaImporter().to_advisory(raw_data)
+    result = [data.to_dict() for data in advisories]
+
+    # TODO: We need to finish this test including the REGEN step.  2022-12-12 Monday 14:48:05.  Done.
+    expected_file = os.path.join(TEST_DATA, f"to-advisory-apache_kafka-expected.json")
+    util_tests.check_results_against_json(result, expected_file)
+
+    # We generate these 2 files solely to vet the output and adjust the importer code.
+    # with open(os.path.join(TEST_DATA, "jmh-test-01.txt"), "w") as f1:
+    #     for advisory_object in result:
+    #         f1.write(f"{advisory_object}\n\n")
+    #         for k, v in advisory_object.items():
+    #             f1.write(f"{k}: {v}\n\n")
+    #         f1.write(f"=================================================\n\n")
+
+    # with open(os.path.join(TEST_DATA, "test-advisories.json"), "w", encoding="utf-8") as f:
+    #     json.dump(result, f, ensure_ascii=False, indent=4)
+
+
+# Check for an unknown CVE value.
+def to_advisory_changed_cve():
+    with open(os.path.join(TEST_DATA, "cve-list-changed-cve.html")) as f:
+        raw_data = f.read()
+    advisories = ApacheKafkaImporter().to_advisory(raw_data)
+
+
+def test_to_advisory_changed_cve_exception():
+    with pytest.raises(KeyError) as excinfo:
+        to_advisory_changed_cve()
+
+    assert "CVE-2022-34918" in str(excinfo.value)
+
+
+# Check for an unknown "Versions affected" value.
+def to_advisory_changed_versions_affected():
+    with open(os.path.join(TEST_DATA, "cve-list-changed-versions-affected.html")) as f:
+        raw_data = f.read()
+    advisories = ApacheKafkaImporter().to_advisory(raw_data)
+
+
+def test_to_advisory_changed_versions_affected_exception():
+    with pytest.raises(KeyError) as excinfo:
+        to_advisory_changed_versions_affected()
+
+    assert "2.8.0 - 2.8.1, 3.0.0 - 3.0.1, 3.1.0 - 3.1.1, 3.2.0 - 3.2.2" in str(excinfo.value)
+
+
+# Check for an unknown "Fixed versions" value.
+def to_advisory_changed_fixed_versions():
+    with open(os.path.join(TEST_DATA, "cve-list-changed-fixed-versions.html")) as f:
+        raw_data = f.read()
+    advisories = ApacheKafkaImporter().to_advisory(raw_data)
+
+
+def test_to_advisory_changed_fixed_versions_exception():
+    with pytest.raises(KeyError) as excinfo:
+        to_advisory_changed_fixed_versions()
+
+    assert "2.8.2, 3.0.2, 3.1.2, 3.2.4" in str(excinfo.value)