From b2ca56b10c47619e546ad743291e0c05f7bb3f38 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:39:57 +0000 Subject: [PATCH 1/4] Initial plan From 3bda8661d9d4c1450e6361c1ec992a196db18775 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:51:04 +0000 Subject: [PATCH 2/4] Integrate openapi-schema-test-suite as external test suite Co-authored-by: p1c2u <1679024+p1c2u@users.noreply.github.com> --- .gitmodules | 3 + pyproject.toml | 1 + tests/test_suite.py | 160 +++++++++++++++++++++++++++++++ vendor/openapi-schema-test-suite | 1 + 4 files changed, 165 insertions(+) create mode 100644 .gitmodules create mode 100644 tests/test_suite.py create mode 160000 vendor/openapi-schema-test-suite diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b7b4487 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendor/openapi-schema-test-suite"] + path = vendor/openapi-schema-test-suite + url = https://github.com/python-openapi/openapi-schema-test-suite.git diff --git a/pyproject.toml b/pyproject.toml index 22db0f3..68c55a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ classifiers = [ ] include = [ {path = "tests", format = "sdist"}, + {path = "vendor/openapi-schema-test-suite/tests", format = "sdist"}, ] [tool.poetry.dependencies] diff --git a/tests/test_suite.py b/tests/test_suite.py new file mode 100644 index 0000000..4078e13 --- /dev/null +++ b/tests/test_suite.py @@ -0,0 +1,160 @@ +""" +Test runner for the openapi-schema-test-suite. + +This module integrates the external test suite from +https://github.com/python-openapi/openapi-schema-test-suite +to validate OAS 3.1 and OAS 3.2 schema validators against +the canonical test cases. +""" +import json +from pathlib import Path +from typing import Any + +import pytest +from jsonschema.exceptions import ValidationError + +from openapi_schema_validator import OAS31Validator +from openapi_schema_validator import OAS32Validator +from openapi_schema_validator import oas31_format_checker +from openapi_schema_validator import oas32_format_checker + +SUITE_ROOT = ( + Path(__file__).parent.parent + / "vendor" + / "openapi-schema-test-suite" + / "tests" +) + +# Known failures due to limitations in the underlying jsonschema library. +# Each entry is (dialect, relative_path, case_description, test_description). +_KNOWN_FAILURES: dict[tuple[str, str, str, str], str] = { + ( + "oas31", + "unevaluated.json", + "unevaluatedProperties with if/then/else", + "non-premium type with name is valid", + ): ( + "jsonschema does not collect annotations from failing 'if' subschemas, " + "causing properties evaluated by 'if' to be reported as unevaluated" + ), + ( + "oas32", + "unevaluated.json", + "unevaluatedProperties with if/then/else", + "non-premium type with name is valid", + ): ( + "jsonschema does not collect annotations from failing 'if' subschemas, " + "causing properties evaluated by 'if' to be reported as unevaluated" + ), + ( + "oas31", + "optional/format/format-assertion.json", + "format uri with assertion", + "a relative URI is not a valid URI", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas31", + "optional/format/format-assertion.json", + "format uri with assertion", + "an invalid URI is not valid", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas32", + "optional/format/format-assertion.json", + "format uri with assertion", + "a relative URI is not a valid URI", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas32", + "optional/format/format-assertion.json", + "format uri with assertion", + "an invalid URI is not valid", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", +} + +_DIALECT_CONFIG: dict[str, dict[str, Any]] = { + "oas31": { + "validator_class": OAS31Validator, + "format_checker": oas31_format_checker, + }, + "oas32": { + "validator_class": OAS32Validator, + "format_checker": oas32_format_checker, + }, +} + + +def _collect_params() -> list[pytest.param]: + params: list[pytest.param] = [] + + for dialect, config in _DIALECT_CONFIG.items(): + dialect_dir = SUITE_ROOT / dialect + if not dialect_dir.is_dir(): + continue + + for json_path in sorted(dialect_dir.rglob("*.json")): + rel_path = json_path.relative_to(dialect_dir) + is_in_optional_dir = rel_path.parts[0] == "optional" + format_checker = config["format_checker"] if is_in_optional_dir else None + + test_cases: list[dict[str, Any]] = json.loads( + json_path.read_text(encoding="utf-8") + ) + for case in test_cases: + case_desc: str = case["description"] + schema: dict[str, Any] = case["schema"] + for test in case["tests"]: + test_desc: str = test["description"] + data: Any = test["data"] + expected_valid: bool = test["valid"] + + param_id = ( + f"{dialect}/{rel_path}/{case_desc}/{test_desc}" + ) + failure_key = ( + dialect, + str(rel_path), + case_desc, + test_desc, + ) + marks: list[pytest.MarkDecorator] = [] + if failure_key in _KNOWN_FAILURES: + marks.append( + pytest.mark.xfail( + reason=_KNOWN_FAILURES[failure_key], + strict=True, + ) + ) + params.append( + pytest.param( + config["validator_class"], + schema, + format_checker, + data, + expected_valid, + id=param_id, + marks=marks, + ) + ) + + return params + + +@pytest.mark.parametrize( + "validator_class,schema,format_checker,data,expected_valid", + _collect_params(), +) +def test_suite( + validator_class: Any, + schema: dict[str, Any], + format_checker: Any, + data: Any, + expected_valid: bool, +) -> None: + validator = validator_class(schema, format_checker=format_checker) + errors = list(validator.iter_errors(data)) + is_valid = len(errors) == 0 + assert is_valid == expected_valid, ( + f"Expected valid={expected_valid}, got valid={is_valid}. " + f"Errors: {[e.message for e in errors]}" + ) diff --git a/vendor/openapi-schema-test-suite b/vendor/openapi-schema-test-suite new file mode 160000 index 0000000..a913d2a --- /dev/null +++ b/vendor/openapi-schema-test-suite @@ -0,0 +1 @@ +Subproject commit a913d2aa71a8dcc9470671520b530e0e92852e9f From c484e9ff813f4249dc9dcbbfa87fe33aa4b96f0f Mon Sep 17 00:00:00 2001 From: p1c2u Date: Fri, 6 Mar 2026 22:35:46 +0000 Subject: [PATCH 3/4] Update test suite --- tests/test_suite.py | 28 +++++----------------------- vendor/openapi-schema-test-suite | 2 +- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/tests/test_suite.py b/tests/test_suite.py index 4078e13..3203074 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -6,6 +6,7 @@ to validate OAS 3.1 and OAS 3.2 schema validators against the canonical test cases. """ + import json from pathlib import Path from typing import Any @@ -25,27 +26,8 @@ / "tests" ) -# Known failures due to limitations in the underlying jsonschema library. # Each entry is (dialect, relative_path, case_description, test_description). _KNOWN_FAILURES: dict[tuple[str, str, str, str], str] = { - ( - "oas31", - "unevaluated.json", - "unevaluatedProperties with if/then/else", - "non-premium type with name is valid", - ): ( - "jsonschema does not collect annotations from failing 'if' subschemas, " - "causing properties evaluated by 'if' to be reported as unevaluated" - ), - ( - "oas32", - "unevaluated.json", - "unevaluatedProperties with if/then/else", - "non-premium type with name is valid", - ): ( - "jsonschema does not collect annotations from failing 'if' subschemas, " - "causing properties evaluated by 'if' to be reported as unevaluated" - ), ( "oas31", "optional/format/format-assertion.json", @@ -95,7 +77,9 @@ def _collect_params() -> list[pytest.param]: for json_path in sorted(dialect_dir.rglob("*.json")): rel_path = json_path.relative_to(dialect_dir) is_in_optional_dir = rel_path.parts[0] == "optional" - format_checker = config["format_checker"] if is_in_optional_dir else None + format_checker = ( + config["format_checker"] if is_in_optional_dir else None + ) test_cases: list[dict[str, Any]] = json.loads( json_path.read_text(encoding="utf-8") @@ -108,9 +92,7 @@ def _collect_params() -> list[pytest.param]: data: Any = test["data"] expected_valid: bool = test["valid"] - param_id = ( - f"{dialect}/{rel_path}/{case_desc}/{test_desc}" - ) + param_id = f"{dialect}/{rel_path}/{case_desc}/{test_desc}" failure_key = ( dialect, str(rel_path), diff --git a/vendor/openapi-schema-test-suite b/vendor/openapi-schema-test-suite index a913d2a..a25f0ac 160000 --- a/vendor/openapi-schema-test-suite +++ b/vendor/openapi-schema-test-suite @@ -1 +1 @@ -Subproject commit a913d2aa71a8dcc9470671520b530e0e92852e9f +Subproject commit a25f0ace12fa075998d880e8b32d314bce823105 From 847e6df3e5a2d81bc22657c62675fcbdf8b225b6 Mon Sep 17 00:00:00 2001 From: p1c2u Date: Sun, 8 Mar 2026 00:25:56 +0000 Subject: [PATCH 4/4] Update test suite 2 --- tests/test_suite.py | 56 +++++++++++++++++++++++++++++++- vendor/openapi-schema-test-suite | 2 +- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tests/test_suite.py b/tests/test_suite.py index 3203074..7bafbbf 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -3,7 +3,7 @@ This module integrates the external test suite from https://github.com/python-openapi/openapi-schema-test-suite -to validate OAS 3.1 and OAS 3.2 schema validators against +to validate OAS 3.0, OAS 3.1 and OAS 3.2 schema validators against the canonical test cases. """ @@ -14,8 +14,10 @@ import pytest from jsonschema.exceptions import ValidationError +from openapi_schema_validator import OAS30Validator from openapi_schema_validator import OAS31Validator from openapi_schema_validator import OAS32Validator +from openapi_schema_validator import oas30_format_checker from openapi_schema_validator import oas31_format_checker from openapi_schema_validator import oas32_format_checker @@ -28,6 +30,18 @@ # Each entry is (dialect, relative_path, case_description, test_description). _KNOWN_FAILURES: dict[tuple[str, str, str, str], str] = { + ( + "oas30", + "optional/format/format-assertion.json", + "format uri with assertion", + "a relative URI is not a valid URI", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas30", + "optional/format/format-assertion.json", + "format uri with assertion", + "an invalid URI is not valid", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", ( "oas31", "optional/format/format-assertion.json", @@ -52,9 +66,49 @@ "format uri with assertion", "an invalid URI is not valid", ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas30", + "discriminator.json", + "discriminator as annotation", + "a cat object is valid", + ): "discriminator not fully supported in base oas30", + ( + "oas30", + "discriminator.json", + "discriminator as annotation", + "a dog object is valid", + ): "discriminator not fully supported in base oas30", + ( + "oas30", + "discriminator.json", + "discriminator with mapping", + "a car object is valid", + ): "discriminator mapping", + ( + "oas30", + "discriminator.json", + "discriminator with mapping", + "a truck object is valid", + ): "discriminator mapping", + ( + "oas30", + "ref.json", + "$ref sibling keywords are ignored", + "a short string is valid because minLength sibling is ignored", + ): "we do not ignore sibling keywords in oas30", + ( + "oas30", + "type.json", + "integer type matches integers", + "a float with zero fractional part is an integer", + ): "float with zero fractional part", } _DIALECT_CONFIG: dict[str, dict[str, Any]] = { + "oas30": { + "validator_class": OAS30Validator, + "format_checker": oas30_format_checker, + }, "oas31": { "validator_class": OAS31Validator, "format_checker": oas31_format_checker, diff --git a/vendor/openapi-schema-test-suite b/vendor/openapi-schema-test-suite index a25f0ac..741c876 160000 --- a/vendor/openapi-schema-test-suite +++ b/vendor/openapi-schema-test-suite @@ -1 +1 @@ -Subproject commit a25f0ace12fa075998d880e8b32d314bce823105 +Subproject commit 741c876969af3a3025dfea4a3f398da23e5e7251