diff --git a/README_COMPATIBILITY_TESTS.md b/README_COMPATIBILITY_TESTS.md new file mode 100644 index 0000000..cafb54c --- /dev/null +++ b/README_COMPATIBILITY_TESTS.md @@ -0,0 +1,515 @@ +# API Compatibility Testing Guide + +## Overview + +The compatibility test suite monitors IonQ API response schemas to detect breaking changes that would affect downstream consumers like qiskit-ionq. Tests warn on schema violations but **do not fail CI** - they only alert teams to potential issues. + +## Quick Start + +### 1. One-Time Setup: Capture Baseline + +```bash +cd /Users/far.mckon/dev/ionq/ionq-core-python + +# Set API key +export IONQ_API_KEY="your-api-key" + +# Capture baseline from current API +python scripts/capture_api_baseline.py \ + --version 1.0.3 \ + --endpoint-user qiskit-ionq + +# Output: tests/compatibility_baselines/qiskit_ionq_v1.0.3.json +``` + +### 2. Run Compatibility Tests + +```bash +# Run all compatibility tests +pytest -m compatibility tests/test_compatibility_qiskit_ionq.py -v -s + +# Run specific test +pytest -m compatibility tests/test_compatibility_qiskit_ionq.py::TestQiskitIonQCompatibilityV1_0_3::test_job_submission_response_schema -v -s +``` + +### 3. Interpret Results + +**✅ All tests PASS** → No breaking changes detected + +**⚠️ Tests SKIP with warnings** → Breaking changes detected: +``` +====================================================================== +API COMPATIBILITY WARNING +====================================================================== +Endpoint User: qiskit-ionq +Version: 1.0.3 +Test: test_job_retrieval_response_schema +Issue: Required field 'metadata' missing from response +====================================================================== +``` + +--- + +## Architecture + +### Components + +``` +ionq-core-python/ +├── tests/ +│ ├── compatibility_conftest.py # Decorator & fixtures +│ ├── compatibility_baselines/ +│ │ └── qiskit_ionq_v1.0.3.json # Expected schemas +│ └── test_compatibility_qiskit_ionq.py # Actual tests +├── scripts/ +│ └── capture_api_baseline.py # Baseline generator +└── pyproject.toml # pytest configuration +``` + +### How It Works + +1. **Baseline Capture**: Script queries live API and saves response schemas +2. **Tests Run**: Compare live API responses against baseline +3. **Decorator**: `@warn_team_on_fail` converts failures to warnings +4. **Output**: Warnings print to stdout, tests skip (not fail) + +--- + +## File Descriptions + +### 1. `compatibility_conftest.py` - Testing Infrastructure + +**Key Components:** + +#### `@warn_team_on_fail` Decorator +```python +@warn_team_on_fail +def test_endpoint(self, ...): + assert "critical_field" in response + # If fails → prints warning, test skips +``` + +Behavior: +- Catches `AssertionError` +- Prints formatted warning with version/endpoint_user +- Marks test as SKIPPED (not FAILED) + +#### `check_schema_compatibility` Fixture +```python +def test_endpoint(check_schema_compatibility): + response = api_call() + check_schema_compatibility("POST /jobs", response, status_code=201) +``` + +Validates: +- Status codes match baseline +- Required fields present +- Critical fields exist (those qiskit-ionq uses) +- Field types match expectations +- Handles nested fields like `metadata.qiskit_header` + +#### `compatibility_baseline` Fixture +```python +@pytest.fixture(scope="session") +def compatibility_baseline(): + """Loads tests/compatibility_baselines/qiskit_ionq_v1.0.3.json""" +``` + +--- + +### 2. `capture_api_baseline.py` - Baseline Generator + +**What It Does:** +1. Connects to IonQ API +2. Submits test Bell circuit +3. Captures response schemas from 9 endpoints +4. Saves to JSON with metadata (version, git commit, timestamp) +5. Cleans up test job + +**Usage:** +```bash +python scripts/capture_api_baseline.py \ + --version 1.0.3 \ + --endpoint-user qiskit-ionq \ + --output custom/path.json # optional +``` + +**Output Format:** +```json +{ + "metadata": { + "version": "1.0.3", + "endpoint_user": "qiskit-ionq", + "captured_at": "2026-06-01T10:00:00Z", + "branch": "main", + "commit": "abc123def" + }, + "endpoints": { + "POST /jobs": { + "response_schema": { ... }, + "critical_fields": ["id", "status"], + "notes": "qiskit-ionq requires..." + }, + ... + } +} +``` + +--- + +### 3. `test_compatibility_qiskit_ionq.py` - Test Suite + +**Structure:** +```python +@pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") +class TestQiskitIonQCompatibilityV1_0_3: + """One test per critical endpoint""" + + @warn_team_on_fail + def test_job_submission_response_schema(self, ...): + """POST /jobs - validate job creation""" + + @warn_team_on_fail + def test_job_retrieval_response_schema(self, ...): + """GET /jobs/{id} - validate job details""" + + # ... 8 more endpoint tests +``` + +**10 Endpoints Tested:** +1. POST /jobs - Job submission +2. GET /jobs/{id} - Job retrieval +3. GET /jobs/{id}/results/probabilities - Results +4. GET /backends/{backend} - Backend info +5. GET /backends/{backend}/characterizations - Calibration +6. GET /jobs/{id}/circuits/{lang} - Compiled circuit +7. PUT /jobs/{id}/status/cancel - Cancel +8. DELETE /jobs/{id} - Delete +9. GET /jobs/estimate - Cost estimation +10. (Metadata and nested fields) + +--- + +## Usage Patterns + +### Pattern 1: On-Demand Testing (Developer) + +```bash +# Before making API changes +pytest -m compatibility tests/test_compatibility_qiskit_ionq.py -v -s + +# Check if any consumers would break +``` + +### Pattern 2: Scheduled Monitoring (CI/CD) + +```yaml +# .github/workflows/compatibility-check.yml +name: API Compatibility Check +on: + schedule: + - cron: '0 0 * * 0' # Weekly + workflow_dispatch: # Manual trigger + +jobs: + compatibility: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run compatibility tests + env: + IONQ_API_KEY: ${{ secrets.IONQ_API_KEY }} + run: | + pytest -m compatibility tests/test_compatibility_qiskit_ionq.py -v -s + continue-on-error: true # Don't fail workflow, just report +``` + +### Pattern 3: Pre-Release Validation + +```bash +# Before deploying API changes +pytest -m compatibility tests/ -v -s > compatibility_report.txt + +# Review warnings +grep "API COMPATIBILITY WARNING" compatibility_report.txt + +# Notify affected teams if warnings found +``` + +### Pattern 4: Multiple Consumer Versions + +```bash +# Add tests for different versions +python scripts/capture_api_baseline.py --version 2.0.0 --endpoint-user qiskit-ionq +python scripts/capture_api_baseline.py --version 1.5.0 --endpoint-user cirq-ionq + +# Test all versions +pytest -m compatibility tests/test_compatibility_*.py -v +``` + +--- + +## Adding Tests for New Consumers + +### Step 1: Capture Baseline + +```bash +python scripts/capture_api_baseline.py \ + --version 0.5.0 \ + --endpoint-user cirq-ionq +``` + +### Step 2: Create Test File + +```python +# tests/test_compatibility_cirq_ionq.py + +@pytest.mark.compatibility(version="0.5.0", endpoint_user="cirq-ionq") +class TestCirqIonQCompatibilityV0_5_0: + + @warn_team_on_fail + def test_job_submission(self, client, check_schema_compatibility): + """Test critical fields for cirq-ionq""" + # ... similar to qiskit-ionq tests +``` + +### Step 3: Document Critical Fields + +In baseline JSON, note which fields cirq-ionq relies on: +```json +{ + "critical_fields": ["job_id", "state"], + "notes": "cirq-ionq uses 'job_id' not 'id'" +} +``` + +--- + +## Configuration + +### pytest.ini / pyproject.toml + +```toml +[tool.pytest.ini_options] +markers = [ + "compatibility: API compatibility monitoring tests", +] +addopts = "-m 'not integration and not compatibility'" # Skip by default +``` + +### Running Only Compatibility Tests + +```bash +# Explicit selection +pytest -m compatibility + +# Exclude from normal runs +pytest # compatibility tests skipped automatically +``` + +--- + +## Maintenance + +### Updating Baselines + +When API changes intentionally and consumer updated: + +```bash +# 1. Capture new baseline +python scripts/capture_api_baseline.py \ + --version 1.0.4 \ + --endpoint-user qiskit-ionq + +# 2. Update test marker +# Edit tests/test_compatibility_qiskit_ionq.py: +@pytest.mark.compatibility(version="1.0.4", endpoint_user="qiskit-ionq") + +# 3. Commit both files +git add tests/compatibility_baselines/qiskit_ionq_v1.0.4.json +git add tests/test_compatibility_qiskit_ionq.py +git commit -m "Update qiskit-ionq compatibility baseline to v1.0.4" +``` + +### Archiving Old Versions + +```python +# Keep for historical reference +@pytest.mark.skip(reason="qiskit-ionq 1.0.3 deprecated") +@pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") +class TestQiskitIonQCompatibilityV1_0_3: + ... +``` + +--- + +## Troubleshooting + +### Issue: "Baseline file not found" + +**Solution:** Run baseline capture +```bash +python scripts/capture_api_baseline.py --version 1.0.3 --endpoint-user qiskit-ionq +``` + +### Issue: "IONQ_API_KEY not set" + +**Solution:** Set environment variable +```bash +export IONQ_API_KEY="your-api-key" +``` + +### Issue: Tests fail (not skip) + +**Possible causes:** +- Test code has syntax error +- Import error (missing dependency) +- Decorator not applied + +**Not expected:** Tests should SKIP with warning, not FAIL + +### Issue: No warnings when expected + +**Check:** +1. Decorator applied: `@warn_team_on_fail` +2. Running with `-s` flag to show stdout +3. Baseline file exists and is loaded + +### Issue: "No completed jobs available" + +**Solution:** Normal for `test_completed_job_results_schema` +- Test will skip if no completed jobs +- Or wait for previous job to complete + +--- + +## Best Practices + +### 1. Capture Baselines from Stable API + +- Run from `main` branch +- After API is deployed +- Before announcing version to consumers + +### 2. Run Tests Regularly + +- **Weekly:** Detect drift early +- **Before releases:** Validate no breaking changes +- **After incidents:** Verify API restored correctly + +### 3. Review Warnings Promptly + +When warnings appear: +1. Identify affected consumer version +2. Check if consumer needs updating +3. Coordinate with consumer team +4. Document breaking change + +### 4. Version Baselines Carefully + +- One baseline per consumer version +- Include git commit in metadata +- Archive old baselines (don't delete) + +### 5. Document Critical Fields + +In baseline JSON, explain why fields are critical: +```json +{ + "critical_fields": ["metadata.qiskit_header"], + "notes": "qiskit-ionq uses metadata.qiskit_header for circuit reconstruction. Removing this breaks job.result()." +} +``` + +--- + +## Examples + +### Example 1: Detect Breaking Change + +```bash +# API removes 'metadata' field from job response +pytest -m compatibility tests/test_compatibility_qiskit_ionq.py -v -s + +# Output: +====================================================================== +API COMPATIBILITY WARNING +====================================================================== +Endpoint User: qiskit-ionq +Version: 1.0.3 +Test: test_job_retrieval_response_schema +Issue: Required field 'metadata' missing from response +====================================================================== + +# Action: Alert qiskit-ionq team before deploying +``` + +### Example 2: Add New Consumer + +```bash +# New consumer: pennylane-ionq v0.1.0 + +# 1. Capture baseline +python scripts/capture_api_baseline.py \ + --version 0.1.0 \ + --endpoint-user pennylane-ionq + +# 2. Create test file +cat > tests/test_compatibility_pennylane_ionq.py << 'EOF' +@pytest.mark.compatibility(version="0.1.0", endpoint_user="pennylane-ionq") +class TestPennylaneIonQCompatibilityV0_1_0: + @warn_team_on_fail + def test_job_submission(self, ...): + ... +EOF + +# 3. Run tests +pytest -m compatibility tests/test_compatibility_pennylane_ionq.py -v -s +``` + +### Example 3: CI Integration + +```yaml +# Slack notification on breaking changes +- name: Check compatibility + run: | + OUTPUT=$(pytest -m compatibility tests/ -v -s 2>&1) + echo "$OUTPUT" + + if echo "$OUTPUT" | grep -q "API COMPATIBILITY WARNING"; then + curl -X POST $SLACK_WEBHOOK \ + -d "{\"text\":\"⚠️ API Breaking Changes Detected\n\`\`\`$OUTPUT\`\`\`\"}" + fi +``` + +--- + +## Summary + +**What This System Provides:** +- ✅ Early warning of breaking API changes +- ✅ Consumer-specific compatibility monitoring +- ✅ Non-blocking warnings (tests don't fail CI) +- ✅ Detailed context (version, endpoint, field) +- ✅ Versioned baselines for historical tracking + +**When to Use:** +- 🔄 Regular monitoring (weekly/monthly) +- 🚀 Before API releases +- 🛠️ During API development +- 📊 Consumer impact analysis + +**Who Benefits:** +- **API Team:** Catch breaking changes before deployment +- **Consumer Teams:** Early notification of incompatibilities +- **QA:** Validate backward compatibility +- **Product:** Coordinate releases across teams + +--- + +## Support + +For questions or issues: +1. Check phase-specific guides: `PHASE1_TESTING.md`, `PHASE2_BASELINE_CAPTURE.md`, `PHASE3_COMPATIBILITY_TESTS.md` +2. Review this README +3. Contact IonQ API team diff --git a/pyproject.toml b/pyproject.toml index 6ec21de..c5b0de6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,9 +89,9 @@ include = ["ionq_core/models/**", "ionq_core/api/**"] invalid-argument-type = "ignore" [tool.pytest.ini_options] -testpaths = ["tests"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" +testpaths = ["tests"] xfail_strict = true filterwarnings = [ "error", @@ -99,8 +99,9 @@ filterwarnings = [ ] markers = [ "integration: marks tests that hit the real IonQ API (deselect with '-m \"not integration\"')", + "compatibility: marks tests that monitor API compatibility for downstream consumers (deselect with '-m \"not compatibility\"')", ] -addopts = "-m 'not integration' --tb=short --cov=ionq_core --cov-fail-under=100" +addopts = "-m 'not integration and not compatibility' --tb=short --cov=ionq_core --cov-fail-under=100" [tool.coverage.run] branch = true diff --git a/scripts/check_ionq-kiskit-1-0-3-vs-live-api.py b/scripts/check_ionq-kiskit-1-0-3-vs-live-api.py new file mode 100755 index 0000000..e04a3d7 --- /dev/null +++ b/scripts/check_ionq-kiskit-1-0-3-vs-live-api.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +Capture API baseline for compatibility testing. + +Runs against the current IonQ API on 'main' branch to establish +expected response schemas for compatibility monitoring. + +Usage: + export IONQ_API_KEY="your-api-key" + python scripts/capture_api_baseline.py --version 1.0.3 --endpoint-user qiskit-ionq +""" + +import argparse +import datetime +import json +import os +import subprocess +import sys +from pathlib import Path + +# Add parent directory to path to import ionq_core +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from ionq_core import IonQClient +from ionq_core.api.backends import get_backend +from ionq_core.api.characterizations import get_characterizations_for_backend +from ionq_core.api.default import ( + create_job, + delete_job, + get_job, +) +from ionq_core.models.circuit_job_creation_payload import CircuitJobCreationPayload + + +def get_git_info(): + """Get current git branch and commit.""" + try: + branch = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], text=True, cwd=Path(__file__).parent.parent + ).strip() + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, cwd=Path(__file__).parent.parent + ).strip() + return branch, commit + except subprocess.CalledProcessError: + return "unknown", "unknown" + + +def capture_job_submission_baseline(client): + """Capture POST /jobs baseline.""" + print(" Creating test job...") + test_circuit = { + "type": "ionq.circuit.v1", + "backend": "simulator", + "shots": 100, + "input": { + "gateset": "qis", + "qubits": 2, + "circuit": [ + {"gate": "h", "target": 0}, + {"gate": "cnot", "control": 0, "target": 1}, + ], + }, + } + + body = CircuitJobCreationPayload.from_dict(test_circuit) + resp = create_job.sync_detailed(client=client, body=body) + + response_dict = resp.parsed.to_dict() + + return { + "request_example": test_circuit, + "response_schema": { + "status_code": resp.status_code.value, + "required_fields": list(response_dict.keys()), + "field_types": {k: type(v).__name__ if v is not None else "NoneType" for k, v in response_dict.items()}, + }, + "critical_fields": ["id", "status"], + "notes": "qiskit-ionq requires 'id' to track job, 'status' for state machine", + }, resp.parsed.id + + +def capture_job_retrieval_baseline(client, job_id): + """Capture GET /jobs/{job_id} baseline.""" + print(f" Retrieving job {job_id}...") + job = get_job.sync(uuid=job_id, client=client) + job_dict = job.to_dict() + + # Extract nested field keys + results_keys = list(job_dict.get("results", {}).keys()) if job_dict.get("results") else [] + metadata_keys = list(job_dict.get("metadata", {}).keys()) if job_dict.get("metadata") else [] + + return { + "response_schema": { + "status_code": 200, + "required_fields": list(job_dict.keys()), + "nested_fields": { + "results": results_keys, + "metadata": metadata_keys, + }, + "field_types": {k: type(v).__name__ if v is not None else "NoneType" for k, v in job_dict.items()}, + }, + "critical_fields": ["id", "status", "results", "metadata"], + "notes": "qiskit-ionq parses metadata for circuit reconstruction", + } + + +def capture_backend_baseline(client): + """Capture GET /backends/{backend} baseline.""" + print(" Getting backend info...") + backend = get_backend.sync("simulator", client=client) + backend_dict = backend.to_dict() + + return { + "response_schema": { + "status_code": 200, + "required_fields": list(backend_dict.keys()), + "field_types": {k: type(v).__name__ if v is not None else "NoneType" for k, v in backend_dict.items()}, + }, + "critical_fields": ["qubits"], + "notes": "qiskit-ionq uses for get_n_qubits() helper", + } + + +def capture_characterization_baseline(client): + """Capture GET /backends/{backend}/characterizations baseline.""" + print(" Getting characterization data...") + try: + resp = get_characterizations_for_backend.sync("qpu.forte-1", client=client, limit=1) + if resp and resp.characterizations: + char = resp.characterizations[0].to_dict() + return { + "response_schema": { + "status_code": 200, + "required_fields": ["characterizations"], + "nested_fields": {"characterizations[]": list(char.keys())}, + }, + "critical_fields": ["characterizations[].qubits", "characterizations[].connectivity"], + "notes": "qiskit-ionq uses for backend.calibration() method", + } + except Exception as e: + print(f" Warning: Could not capture characterization baseline: {e}") + + # Fallback minimal schema + return { + "response_schema": { + "status_code": 200, + "required_fields": ["characterizations"], + }, + "critical_fields": [], + "notes": "Baseline capture failed - using minimal schema", + } + + +def main(): + parser = argparse.ArgumentParser(description="Capture API baseline for compatibility testing") + parser.add_argument("--version", required=True, help="Version to capture baseline for (e.g., 1.0.3)") + parser.add_argument("--endpoint-user", required=True, help="Endpoint user name (e.g., qiskit-ionq)") + parser.add_argument( + "--output", help="Output file path (default: tests/compatibility_baselines/{endpoint_user}_v{version}.json)" + ) + + args = parser.parse_args() + + # Get API key + api_key = os.environ.get("IONQ_API_KEY") + if not api_key: + print("Error: IONQ_API_KEY environment variable not set") + print("Set it with: export IONQ_API_KEY='your-api-key'") + return 1 + + client = IonQClient(api_key=api_key) + + # Get git info + branch, commit = get_git_info() + + print(f"\n{'=' * 70}") + print("Capturing API Baseline") + print(f"{'=' * 70}") + print(f"Endpoint User: {args.endpoint_user}") + print(f"Version: {args.version}") + print(f"Git Branch: {branch}") + print(f"Git Commit: {commit[:8]}") + print(f"{'=' * 70}\n") + + # Initialize baseline structure + baseline = { + "metadata": { + "version": args.version, + "endpoint_user": args.endpoint_user, + "captured_at": datetime.datetime.utcnow().isoformat() + "Z", + "api_version": "v0.4", + "branch": branch, + "commit": commit, + }, + "endpoints": {}, + } + + test_job_id = None + + try: + # 1. POST /jobs + print("1. Capturing POST /jobs...") + job_baseline, test_job_id = capture_job_submission_baseline(client) + baseline["endpoints"]["POST /jobs"] = job_baseline + + # 2. GET /jobs/{job_id} + print("2. Capturing GET /jobs/{job_id}...") + baseline["endpoints"]["GET /jobs/{job_id}"] = capture_job_retrieval_baseline(client, test_job_id) + + # 3. GET /backends/{backend} + print("3. Capturing GET /backends/{backend}...") + baseline["endpoints"]["GET /backends/{backend}"] = capture_backend_baseline(client) + + # 4. GET /backends/{backend}/characterizations + print("4. Capturing GET /backends/{backend}/characterizations...") + baseline["endpoints"]["GET /backends/{backend}/characterizations"] = capture_characterization_baseline(client) + + # 5. Add remaining endpoints with manual schemas (no live capture needed) + print("5. Adding manual schemas for remaining endpoints...") + + baseline["endpoints"]["GET /jobs/{job_id}/results/probabilities"] = { + "response_schema": { + "status_code": 200, + "response_type": "object", + "value_constraints": {"all_values": "float", "range": [0.0, 1.0]}, + }, + "critical_fields": ["*"], + "notes": "qiskit-ionq expects dict[str, float] with keys as bitstrings", + } + + baseline["endpoints"]["GET /jobs/{job_id}/circuits/{lang}"] = { + "response_schema": { + "status_code": 200, + "response_type": "string", + }, + "critical_fields": None, + "notes": "qiskit-ionq.compiled_circuit() returns this directly", + } + + baseline["endpoints"]["PUT /jobs/{job_id}/status/cancel"] = { + "response_schema": {"status_code": 200, "required_fields": ["id", "status"]}, + "critical_fields": ["status"], + "notes": "qiskit-ionq checks status after cancel", + } + + baseline["endpoints"]["DELETE /jobs/{job_id}"] = { + "response_schema": {"status_code": 200, "required_fields": ["id"]}, + "critical_fields": ["id"], + "notes": "qiskit-ionq verifies deletion by ID", + } + + baseline["endpoints"]["GET /jobs/estimate"] = { + "response_schema": { + "status_code": 200, + "required_fields": ["estimated_cost", "cost_unit", "estimated_execution_time"], + }, + "critical_fields": ["estimated_cost", "estimated_execution_time"], + "notes": "qiskit-ionq displays cost estimates to users", + } + + finally: + # Cleanup: Delete test job + if test_job_id: + try: + print(f"\nCleaning up: Deleting test job {test_job_id}...") + delete_job.sync(uuid=test_job_id, client=client) + except Exception as e: + print(f" Warning: Could not delete test job: {e}") + + # Save baseline + if args.output: + output_path = Path(args.output) + else: + output_path = ( + Path(__file__).parent.parent + / "tests" + / "compatibility_baselines" + / f"{args.endpoint_user}_v{args.version}.json" + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, "w") as f: + json.dump(baseline, f, indent=2) + + print(f"\n{'=' * 70}") + print("✅ Baseline Saved") + print(f"{'=' * 70}") + print(f"Location: {output_path}") + print(f"Endpoints Captured: {len(baseline['endpoints'])}") + print(f"{'=' * 70}\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/PHASE1_TESTING.md b/tests/PHASE1_TESTING.md new file mode 100644 index 0000000..87c6cbe --- /dev/null +++ b/tests/PHASE1_TESTING.md @@ -0,0 +1,125 @@ +# Phase 1: Testing the Decorator + +## What We've Created + +1. **`compatibility_conftest.py`** - Core decorator and fixtures + - `@warn_team_on_fail` decorator + - `compatibility_baseline` fixture (loads JSON) + - `check_schema_compatibility` fixture (validates schemas) + - Pytest hooks for marker injection + +2. **`test_compatibility_decorator.py`** - Tests to verify decorator works + - Passing tests work normally + - Failing tests print warnings and skip + - Version/endpoint_user extracted from marker + +3. **`pyproject.toml`** - Updated configuration + - Added `compatibility` marker + - Excluded compatibility tests by default + - Added CompatibilityWarning to filterwarnings + +## Testing Instructions + +### Step 1: Verify Decorator Behavior + +Run the decorator tests to see warnings in action: + +```bash +cd /Users/far.mckon/dev/ionq/ionq-core-python + +# Run compatibility tests with verbose output +pytest -m compatibility tests/test_compatibility_decorator.py -v -s + +# Expected output: +# - test_passing_test_unchanged: PASSED (no warning) +# - test_failing_test_warns_and_skips: SKIPPED (prints warning) +# - test_missing_field_warns: SKIPPED (prints warning) +# - test_type_mismatch_warns: SKIPPED (prints warning) +``` + +### Step 2: Verify Warning Format + +Look for output like: + +``` +====================================================================== +API COMPATIBILITY WARNING +====================================================================== +Endpoint User: qiskit-ionq +Version: 1.0.3 +Test: test_failing_test_warns_and_skips +Issue: This is a test breaking change +====================================================================== +``` + +### Step 3: Verify Default Behavior + +Confirm compatibility tests are skipped by default: + +```bash +# Run all tests without -m flag +pytest tests/test_compatibility_decorator.py -v + +# Should output: +# "SKIPPED [5] pyproject.toml:103: Skipped: not in the list of explicitly selected markers" +``` + +### Step 4: Verify Normal Tests Unaffected + +```bash +# Run the non-decorated test +pytest tests/test_compatibility_decorator.py::test_without_decorator -v + +# Should PASS normally +``` + +## Expected Results + +✅ **Success Criteria:** +1. Passing tests with decorator work normally +2. Failing tests with decorator print warnings and skip +3. Warning message includes version and endpoint_user +4. Compatibility tests excluded from default test run +5. Regular tests without decorator unaffected + +❌ **Failure Cases to Watch For:** +- Decorator causes passing tests to fail +- Warnings not printed to stdout +- Version/endpoint_user not extracted from marker +- Regular tests affected by decorator + +## Troubleshooting + +### "No module named 'compatibility_conftest'" + +Make sure you're running from the repository root: +```bash +cd /Users/far.mckon/dev/ionq/ionq-core-python +``` + +### Warnings not showing + +Add `-s` flag to show stdout: +```bash +pytest -m compatibility tests/test_compatibility_decorator.py -v -s +``` + +### Import errors + +Install dev dependencies: +```bash +pip install -e .[dev] +# or with Poetry +poetry install +``` + +## Next Steps + +Once Phase 1 tests pass, we'll move to: + +**Phase 2: Baseline Capture** +- Create `scripts/capture_api_baseline.py` +- Run against live API to capture schemas +- Generate `tests/compatibility_baselines/qiskit_ionq_v1.0.3.json` + +Let me know when Phase 1 tests are working! diff --git a/tests/compatibility_conftest.py b/tests/compatibility_conftest.py new file mode 100644 index 0000000..de017c1 --- /dev/null +++ b/tests/compatibility_conftest.py @@ -0,0 +1,241 @@ +""" +Compatibility testing fixtures and decorators. + +This module provides infrastructure for API compatibility monitoring tests +that warn on breaking changes without failing the test suite. +""" + +import functools +import json +import warnings +from collections.abc import Callable +from pathlib import Path + +import pytest + + +class CompatibilityWarning(UserWarning): + """Warning category for API compatibility issues.""" + + pass + + +def warn_team_instead_of_fail(team_name: str) -> Callable: + """ + Decorator factory that catches AssertionErrors and converts them to warnings. + + Allows compatibility tests to detect breaking changes without failing + the test suite. Prints warning with version, endpoint_user, and team context. + + Usage: + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_response_schema(self, ...): + assert "required_field" in response + + The decorator will: + 1. Try to run the test normally + 2. If AssertionError occurs, print formatted warning to stdout + 3. Issue Python warning for test runners that capture warnings + 4. Skip the test (mark as passed) instead of failing + + Args: + team_name: Name of the team to notify on failure + + Returns: + Decorator function that wraps test functions + """ + + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args, **kwargs): + # Extract test context from pytest markers + # This is set by pytest_collection_modifyitems hook below + test_obj = args[0] if args else None + version = getattr(test_obj, "_compatibility_version", "unknown") + endpoint_user = getattr(test_obj, "_compatibility_endpoint_user", "unknown") + + try: + return func(*args, **kwargs) + except AssertionError as e: + # Format warning message for stdout + warning_msg = ( + f"\n{'=' * 70}\n" + f"API COMPATIBILITY WARNING\n" + f"{'=' * 70}\n" + f"Team: {team_name}\n" + f"Endpoint User: {endpoint_user}\n" + f"Version: {version}\n" + f"Test: {func.__name__}\n" + f"Issue: {e!s}\n" + f"{'=' * 70}\n" + ) + + # Print to stdout (visible in test output) + print(warning_msg) + + # Also issue Python warning for test runners that capture warnings + warnings.warn( + f"[{team_name}] [{endpoint_user} v{version}] {func.__name__}: {e}", + CompatibilityWarning, + stacklevel=2, + ) + + # Skip test instead of failing - we only want to warn + pytest.skip(f"Compatibility issue detected: {e}") + + return wrapper + + return decorator + + +@pytest.fixture(scope="session") +def compatibility_baseline(): + """ + Load compatibility baseline JSON for schema comparison. + + Returns: + dict: Parsed baseline containing endpoint schemas + + Raises: + pytest.skip: If baseline file not found + """ + baseline_path = Path(__file__).parent / "compatibility_baselines" / "qiskit_ionq_v1.0.3.json" + + if not baseline_path.exists(): + pytest.skip(f"Baseline file not found: {baseline_path}") + + with open(baseline_path) as f: + return json.load(f) + + +@pytest.fixture +def check_schema_compatibility(compatibility_baseline): + """ + Fixture providing schema compatibility checking function. + + Returns a callable that compares actual API responses against + the baseline schema expectations. + + Usage: + def test_endpoint(check_schema_compatibility): + response = api_call() + check_schema_compatibility("POST /jobs", response, status_code=201) + + Returns: + Callable[[str, dict, int], None]: Schema checker function + """ + + def _check(endpoint: str, actual_response: dict, status_code: int = 200): + """ + Compare actual response against baseline schema. + + Args: + endpoint: API endpoint identifier (e.g., "POST /jobs") + actual_response: Actual API response dict + status_code: Actual HTTP status code + + Raises: + AssertionError: If breaking change detected + UserWarning: If baseline not found for endpoint + """ + baseline = compatibility_baseline["endpoints"].get(endpoint) + if not baseline: + warnings.warn(f"No baseline found for endpoint: {endpoint}", CompatibilityWarning, stacklevel=1) + return + + schema = baseline["response_schema"] + + # Check status code + expected_status = schema.get("status_code") + if expected_status is not None: + assert status_code == expected_status, f"Status code changed: expected {expected_status}, got {status_code}" + + # Check required fields exist + required = schema.get("required_fields", []) + for field in required: + assert field in actual_response, f"Required field '{field}' missing from response" + + # Check critical fields (those qiskit-ionq actually uses) + critical = baseline.get("critical_fields", []) + for field_path in critical: + if field_path == "*": + # Wildcard - all fields are critical, just verify response exists + continue + + # Support nested field paths like "metadata.qiskit_header" + parts = field_path.split(".") + current = actual_response + + for part in parts: + # Handle array notation like "characterizations[]" + if part.endswith("[]"): + part = part[:-2] + assert isinstance(current.get(part), list), f"Field '{field_path}' should be a list" + if current[part]: # Check first element if list non-empty + current = current[part][0] + else: + break # Empty list, can't check nested fields + else: + assert part in current, f"Critical field '{field_path}' missing from response" + current = current.get(part) + if current is None: + break # Null value, can't check nested fields + + # Check field types for critical fields + field_types = schema.get("field_types", {}) + for field, expected_type in field_types.items(): + if field not in actual_response: + # Field not present - may be nullable + continue + + actual_value = actual_response[field] + + # Handle nullable types like "string|null" + allowed_types = expected_type.split("|") + type_map = { + "string": str, + "integer": int, + "float": float, + "boolean": bool, + "object": dict, + "array": list, + "null": type(None), + } + + type_matches = any(isinstance(actual_value, type_map[t]) for t in allowed_types if t in type_map) + + assert type_matches, ( + f"Field '{field}' type mismatch: expected {expected_type}, got {type(actual_value).__name__}" + ) + + return _check + + +def pytest_configure(config): + """Register compatibility marker with pytest.""" + config.addinivalue_line("markers", "compatibility(version, endpoint_user): mark test as API compatibility check") + + +def pytest_collection_modifyitems(config, items): + """ + Inject version and endpoint_user into test instances from markers. + + This hook runs during test collection and extracts the version and + endpoint_user parameters from @pytest.mark.compatibility markers, + storing them on the test instance for access by @warn_team_on_fail. + + Args: + config: Pytest config object + items: List of collected test items + """ + for item in items: + marker = item.get_closest_marker("compatibility") + if marker: + version = marker.kwargs.get("version", "unknown") + endpoint_user = marker.kwargs.get("endpoint_user", "unknown") + + # Store in test instance for decorator access + if hasattr(item, "instance") and item.instance: + item.instance._compatibility_version = version + item.instance._compatibility_endpoint_user = endpoint_user diff --git a/tests/test_compatibility_decorator.py b/tests/test_compatibility_decorator.py new file mode 100644 index 0000000..6879f66 --- /dev/null +++ b/tests/test_compatibility_decorator.py @@ -0,0 +1,66 @@ +""" +Test the compatibility decorator behavior. + +This test file verifies that @warn_team_instead_of_fail works correctly: +1. Catching assertion failures +2. Printing warnings with version/endpoint_user/team context +3. Skipping tests instead of failing + +Run with: pytest tests/test_compatibility_decorator.py -v -s +""" + +import pytest + +from .compatibility_conftest import warn_team_instead_of_fail + + +@pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") +class TestDecoratorBehavior: + """Test suite to verify decorator functionality.""" + + @warn_team_instead_of_fail(team_name="devtools") + def test_passing_test_unchanged(self): + """Passing tests should work normally.""" + assert True + assert 1 + 1 == 2 + + @warn_team_instead_of_fail(team_name="devtools") + def test_failing_test_warns_and_skips(self): + """Failing tests should print warning and skip.""" + # This assertion will fail + assert not True, "This is a test breaking change" + + @warn_team_instead_of_fail(team_name="devtools") + def test_missing_field_warns(self): + """Simulate missing field in API response.""" + response = {"id": "job-123", "status": "completed"} + # This will fail - 'metadata' is missing + assert "metadata" in response, "Required field 'metadata' missing from response" + + @warn_team_instead_of_fail(team_name="devtools") + def test_type_mismatch_warns(self): + """Simulate type change in API response.""" + response = {"qubits": "25"} # Should be int, but is string + assert isinstance(response["qubits"], int), ( + f"Field 'qubits' type mismatch: expected int, got {type(response['qubits']).__name__}" + ) + + +@pytest.mark.compatibility(version="2.0.0", endpoint_user="cirq-ionq") +class TestMultipleConsumers: + """Verify decorator works with different version/consumer combinations.""" + + @warn_team_instead_of_fail(team_name="devtools") + def test_cirq_ionq_compatibility(self): + """Test with different endpoint_user.""" + response = {"job_id": "123"} # cirq-ionq expects 'job_id' not 'id' + assert "id" in response, "cirq-ionq v2.0.0 expects 'id' field" + + +def test_without_decorator(): + """ + Regular test without decorator should fail normally. + + This verifies that non-compatibility tests aren't affected. + """ + assert True # This test should pass normally diff --git a/tests/test_compatibility_qiskit_ionq.py b/tests/test_compatibility_qiskit_ionq.py new file mode 100644 index 0000000..f29c05b --- /dev/null +++ b/tests/test_compatibility_qiskit_ionq.py @@ -0,0 +1,350 @@ +""" +API compatibility tests for qiskit-ionq v1.0.3. + +These tests monitor API response schemas to detect breaking changes +that would affect qiskit-ionq. Tests do not fail - they only warn. + +Run with: pytest -m compatibility tests/test_compatibility_qiskit_ionq.py -v -s +""" + +import pytest + +from ionq_core.api.backends import get_backend +from ionq_core.api.characterizations import get_characterizations_for_backend +from ionq_core.api.default import ( + cancel_job, + create_job, + delete_job, + estimate_job_cost, + get_compiled_file, + get_job, + get_job_probabilities, + get_jobs, +) +from ionq_core.models.circuit_job_creation_payload import CircuitJobCreationPayload + +# Import compatibility fixtures +from .compatibility_conftest import warn_team_instead_of_fail + + +class TestQiskitIonQCompatibilityV1_0_3: + """ + Compatibility tests for qiskit-ionq v1.0.3. + + These tests verify that API response schemas contain all fields + that qiskit-ionq relies on. Breaking changes trigger warnings + but do not fail the test suite. + + Each test is marked with @warn_team_instead_of_fail(team_name="devtools") + which converts assertion failures to warnings instead of test failures. + """ + + @pytest.fixture(scope="class") + def bell_circuit(self): + """Standard Bell state circuit for testing.""" + return { + "type": "ionq.circuit.v1", + "backend": "simulator", + "shots": 100, + "input": { + "gateset": "qis", + "qubits": 2, + "circuit": [ + {"gate": "h", "target": 0}, + {"gate": "cnot", "control": 0, "target": 1}, + ], + }, + } + + @pytest.fixture(scope="class") + def test_job_id(self, client, bell_circuit): + """ + Create a test job and return its ID. + + This job will be used across multiple tests in the class. + It's automatically cleaned up by the integration test fixtures. + """ + body = CircuitJobCreationPayload.from_dict(bell_circuit) + result = create_job.sync(client=client, body=body) + return result.id + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_job_submission_response_schema(self, client, bell_circuit, check_schema_compatibility): + """ + POST /jobs - Verify job submission response contains required fields. + + qiskit-ionq relies on: + - 'id': To track job throughout lifecycle + - 'status': Initial job state (should be 'submitted') + + Breaking changes: + - Removing 'id' field would break job tracking + - Changing status code from 201 would break client code + - Removing 'status' field would break status checks + """ + body = CircuitJobCreationPayload.from_dict(bell_circuit) + resp = create_job.sync_detailed(client=client, body=body) + + # Check against baseline + check_schema_compatibility("POST /jobs", resp.parsed.to_dict(), status_code=resp.status_code.value) + + # Explicit critical field checks + assert resp.parsed.id is not None, "Job ID is None" + assert resp.parsed.status is not None, "Job status is None" + assert resp.status_code.value == 201, f"Expected status 201, got {resp.status_code.value}" + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_job_retrieval_response_schema(self, client, test_job_id, check_schema_compatibility): + """ + GET /jobs/{job_id} - Verify job retrieval response schema. + + qiskit-ionq relies on: + - 'id', 'status', 'backend', 'type': Basic job info + - 'results': Contains probabilities/shots URLs + - 'metadata': Contains qiskit_header for circuit reconstruction + - 'execution_duration_ms': For timing info + + Breaking changes: + - Removing 'metadata' would break circuit reconstruction + - Removing 'results' would break result retrieval + - Changing field types would break parsing + """ + job = get_job.sync(uuid=test_job_id, client=client) + job_dict = job.to_dict() + + check_schema_compatibility("GET /jobs/{job_id}", job_dict, status_code=200) + + # Critical fields for qiskit-ionq + assert "id" in job_dict, "Missing 'id' field" + assert "status" in job_dict, "Missing 'status' field" + assert "backend" in job_dict, "Missing 'backend' field" + assert "type" in job_dict, "Missing 'type' field" + + # These can be None but key must exist + assert "metadata" in job_dict or job_dict.get("metadata") is None + assert "results" in job_dict or job_dict.get("results") is None + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_completed_job_results_schema(self, client, check_schema_compatibility): + """ + GET /jobs/{job_id}/results/probabilities - Verify results format. + + qiskit-ionq expects: + - Dict[str, float] where keys are bitstrings (e.g., "00", "11") + - All values between 0.0 and 1.0 + - Keys should be decimal integers as strings + + Breaking changes: + - Changing to different format would break result parsing + - Values outside [0,1] would indicate broken probability distribution + - Non-numeric keys would break bitstring parsing + """ + # Use an already-completed job to avoid timeout + resp = get_jobs.sync(client=client, status="completed", limit=1) + + if not resp or not resp.jobs: + pytest.skip("No completed jobs available for testing") + + job_id = resp.jobs[0].id + probs = get_job_probabilities.sync(uuid=job_id, client=client) + + if not probs: + pytest.skip("No probabilities available for completed job") + + probs_dict = probs.additional_properties + + # Verify it's a dict + assert isinstance(probs_dict, dict), "Results should be a dictionary" + + # Verify all values are floats between 0 and 1 + for key, value in probs_dict.items(): + assert isinstance(key, str), f"Key {key} should be string" + assert isinstance(value, (int, float)), f"Value for {key} should be numeric" + assert 0.0 <= value <= 1.0, f"Probability {value} out of range [0, 1]" + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_backend_info_schema(self, client, check_schema_compatibility): + """ + GET /backends/{backend} - Verify backend info schema. + + qiskit-ionq relies on: + - 'qubits': Number of qubits (used in get_n_qubits helper) + - 'status': Backend availability + - 'backend': Backend name + + Breaking changes: + - Removing 'qubits' would break circuit validation + - Changing 'qubits' type to non-int would break numeric operations + """ + backend = get_backend.sync("simulator", client=client) + backend_dict = backend.to_dict() + + check_schema_compatibility("GET /backends/{backend}", backend_dict, status_code=200) + + # Critical field + assert "qubits" in backend_dict, "Missing 'qubits' field" + assert isinstance(backend_dict["qubits"], int), f"'qubits' should be int, got {type(backend_dict['qubits'])}" + assert backend_dict["qubits"] > 0, f"'qubits' should be positive, got {backend_dict['qubits']}" + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_characterization_schema(self, client, check_schema_compatibility): + """ + GET /backends/{backend}/characterizations - Verify calibration data schema. + + qiskit-ionq relies on: + - 'characterizations' array + - Each entry: 'qubits', 'connectivity', 'fidelity', 'timing' + - Used for backend.calibration() method + + Breaking changes: + - Removing 'characterizations' array would break calibration data access + - Removing 'qubits' or 'connectivity' would break topology info + """ + try: + resp = get_characterizations_for_backend.sync("qpu.forte-1", client=client, limit=1) + except Exception as e: + pytest.skip(f"Characterizations endpoint not accessible: {e}") + + if not resp or not resp.characterizations: + pytest.skip("No characterizations available") + + char = resp.characterizations[0] + char_dict = char.to_dict() + + # Verify critical fields exist + assert "qubits" in char_dict, "Missing 'qubits' in characterization" + assert "connectivity" in char_dict or char_dict.get("connectivity") is None + assert isinstance(char_dict["qubits"], int), "'qubits' should be int" + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_compiled_circuit_schema(self, client, test_job_id): + """ + GET /jobs/{job_id}/circuits/{lang} - Verify compiled circuit format. + + qiskit-ionq expects: + - String response + - 'native': JSON-parseable circuit + - 'qasm3': OpenQASM 3 source + + Breaking changes: + - Changing response type from string would break compiled_circuit() method + - Invalid JSON for 'native' would break parsing + """ + import json as json_module + + try: + # Try to get compiled circuit (may not be available immediately) + result = get_compiled_file.sync(uuid=test_job_id, lang="native", client=client) + + if result: + assert isinstance(result, str), "Compiled circuit should be string" + + # Try parsing as JSON for native format + try: + parsed = json_module.loads(result) + assert isinstance(parsed, (list, dict)), "Native circuit should be JSON array or object" + except json_module.JSONDecodeError: + # May not be compiled yet or different format + pass + except Exception as e: + # Compiled circuit might not be available for all jobs + pytest.skip(f"Compiled circuit not available: {e}") + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_job_cancellation_schema(self, client, bell_circuit): + """ + PUT /jobs/{job_id}/status/cancel - Verify cancellation response. + + qiskit-ionq checks 'status' after cancellation. + + Breaking changes: + - Removing 'status' field would break cancel verification + - Removing 'id' field would break job identification + """ + # Create job to cancel + body = CircuitJobCreationPayload.from_dict(bell_circuit) + result = create_job.sync(client=client, body=body) + job_id = result.id + + try: + # Cancel immediately + cancel_result = cancel_job.sync(uuid=job_id, client=client) + + if cancel_result: + cancel_dict = cancel_result.to_dict() + assert "id" in cancel_dict, "Missing 'id' in cancel response" + assert "status" in cancel_dict, "Missing 'status' in cancel response" + finally: + # Clean up - delete the job + try: + delete_job.sync(uuid=job_id, client=client) + except Exception as e: + print(f"Error occurred while deleting job: {e}") + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_job_deletion_schema(self, client, bell_circuit): + """ + DELETE /jobs/{job_id} - Verify deletion response. + + qiskit-ionq verifies deletion by checking returned ID. + + Breaking changes: + - Removing 'id' field would break deletion verification + - Changing status code would break error handling + """ + # Create job to delete + body = CircuitJobCreationPayload.from_dict(bell_circuit) + result = create_job.sync(client=client, body=body) + job_id = result.id + + # Delete + delete_result = delete_job.sync(uuid=job_id, client=client) + + if delete_result: + delete_dict = delete_result.to_dict() + assert "id" in delete_dict, "Missing 'id' in delete response" + + @pytest.mark.compatibility(version="1.0.3", endpoint_user="qiskit-ionq") + @warn_team_instead_of_fail(team_name="devtools") + def test_cost_estimation_schema(self, client, check_schema_compatibility): + """ + GET /jobs/estimate - Verify cost estimation response. + + qiskit-ionq displays: + - 'estimated_cost': Cost value + - 'estimated_execution_time': Time estimate + + Breaking changes: + - Removing cost fields would break cost display + - Changing types would break numeric calculations + """ + result = estimate_job_cost.sync( + client=client, + backend="qpu.forte-1", + qubits=2, + shots=1000, + field_1q_gates=1, + field_2q_gates=1, + ) + + if not result: + pytest.skip("Cost estimation not available") + + result_dict = result.to_dict() + + # Check critical fields + assert "estimated_cost" in result_dict, "Missing 'estimated_cost'" + assert "estimated_execution_time" in result_dict, "Missing 'estimated_execution_time'" + + # These should be numeric + cost = result_dict.get("estimated_cost") + if cost is not None: + assert isinstance(cost, (int, float)), f"'estimated_cost' should be numeric, got {type(cost)}"