diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b7b4487 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendor/openapi-schema-test-suite"] + path = vendor/openapi-schema-test-suite + url = https://github.com/python-openapi/openapi-schema-test-suite.git diff --git a/pyproject.toml b/pyproject.toml index 22db0f3..68c55a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ classifiers = [ ] include = [ {path = "tests", format = "sdist"}, + {path = "vendor/openapi-schema-test-suite/tests", format = "sdist"}, ] [tool.poetry.dependencies] diff --git a/tests/test_suite.py b/tests/test_suite.py new file mode 100644 index 0000000..7bafbbf --- /dev/null +++ b/tests/test_suite.py @@ -0,0 +1,196 @@ +""" +Test runner for the openapi-schema-test-suite. + +This module integrates the external test suite from +https://github.com/python-openapi/openapi-schema-test-suite +to validate OAS 3.0, OAS 3.1 and OAS 3.2 schema validators against +the canonical test cases. +""" + +import json +from pathlib import Path +from typing import Any + +import pytest +from jsonschema.exceptions import ValidationError + +from openapi_schema_validator import OAS30Validator +from openapi_schema_validator import OAS31Validator +from openapi_schema_validator import OAS32Validator +from openapi_schema_validator import oas30_format_checker +from openapi_schema_validator import oas31_format_checker +from openapi_schema_validator import oas32_format_checker + +SUITE_ROOT = ( + Path(__file__).parent.parent + / "vendor" + / "openapi-schema-test-suite" + / "tests" +) + +# Each entry is (dialect, relative_path, case_description, test_description). +_KNOWN_FAILURES: dict[tuple[str, str, str, str], str] = { + ( + "oas30", + "optional/format/format-assertion.json", + "format uri with assertion", + "a relative URI is not a valid URI", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas30", + "optional/format/format-assertion.json", + "format uri with assertion", + "an invalid URI is not valid", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas31", + "optional/format/format-assertion.json", + "format uri with assertion", + "a relative URI is not a valid URI", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas31", + "optional/format/format-assertion.json", + "format uri with assertion", + "an invalid URI is not valid", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas32", + "optional/format/format-assertion.json", + "format uri with assertion", + "a relative URI is not a valid URI", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas32", + "optional/format/format-assertion.json", + "format uri with assertion", + "an invalid URI is not valid", + ): "uri format checker does not validate RFC 3986 absolute-URI requirement", + ( + "oas30", + "discriminator.json", + "discriminator as annotation", + "a cat object is valid", + ): "discriminator not fully supported in base oas30", + ( + "oas30", + "discriminator.json", + "discriminator as annotation", + "a dog object is valid", + ): "discriminator not fully supported in base oas30", + ( + "oas30", + "discriminator.json", + "discriminator with mapping", + "a car object is valid", + ): "discriminator mapping", + ( + "oas30", + "discriminator.json", + "discriminator with mapping", + "a truck object is valid", + ): "discriminator mapping", + ( + "oas30", + "ref.json", + "$ref sibling keywords are ignored", + "a short string is valid because minLength sibling is ignored", + ): "we do not ignore sibling keywords in oas30", + ( + "oas30", + "type.json", + "integer type matches integers", + "a float with zero fractional part is an integer", + ): "float with zero fractional part", +} + +_DIALECT_CONFIG: dict[str, dict[str, Any]] = { + "oas30": { + "validator_class": OAS30Validator, + "format_checker": oas30_format_checker, + }, + "oas31": { + "validator_class": OAS31Validator, + "format_checker": oas31_format_checker, + }, + "oas32": { + "validator_class": OAS32Validator, + "format_checker": oas32_format_checker, + }, +} + + +def _collect_params() -> list[pytest.param]: + params: list[pytest.param] = [] + + for dialect, config in _DIALECT_CONFIG.items(): + dialect_dir = SUITE_ROOT / dialect + if not dialect_dir.is_dir(): + continue + + for json_path in sorted(dialect_dir.rglob("*.json")): + rel_path = json_path.relative_to(dialect_dir) + is_in_optional_dir = rel_path.parts[0] == "optional" + format_checker = ( + config["format_checker"] if is_in_optional_dir else None + ) + + test_cases: list[dict[str, Any]] = json.loads( + json_path.read_text(encoding="utf-8") + ) + for case in test_cases: + case_desc: str = case["description"] + schema: dict[str, Any] = case["schema"] + for test in case["tests"]: + test_desc: str = test["description"] + data: Any = test["data"] + expected_valid: bool = test["valid"] + + param_id = f"{dialect}/{rel_path}/{case_desc}/{test_desc}" + failure_key = ( + dialect, + str(rel_path), + case_desc, + test_desc, + ) + marks: list[pytest.MarkDecorator] = [] + if failure_key in _KNOWN_FAILURES: + marks.append( + pytest.mark.xfail( + reason=_KNOWN_FAILURES[failure_key], + strict=True, + ) + ) + params.append( + pytest.param( + config["validator_class"], + schema, + format_checker, + data, + expected_valid, + id=param_id, + marks=marks, + ) + ) + + return params + + +@pytest.mark.parametrize( + "validator_class,schema,format_checker,data,expected_valid", + _collect_params(), +) +def test_suite( + validator_class: Any, + schema: dict[str, Any], + format_checker: Any, + data: Any, + expected_valid: bool, +) -> None: + validator = validator_class(schema, format_checker=format_checker) + errors = list(validator.iter_errors(data)) + is_valid = len(errors) == 0 + assert is_valid == expected_valid, ( + f"Expected valid={expected_valid}, got valid={is_valid}. " + f"Errors: {[e.message for e in errors]}" + ) diff --git a/vendor/openapi-schema-test-suite b/vendor/openapi-schema-test-suite new file mode 160000 index 0000000..741c876 --- /dev/null +++ b/vendor/openapi-schema-test-suite @@ -0,0 +1 @@ +Subproject commit 741c876969af3a3025dfea4a3f398da23e5e7251