test(eval): add external-context regression fixtures

Author: haasonsaas

TODO.md

@@ -139,7 +139,7 @@ This roadmap is derived from deep research into Greptile's public docs, blog, MC
 
 ## 10. Eval, Benchmarking, and Model Governance
 
-91. [ ] Add eval fixtures for external-context alignment, not just diff-local correctness.
+91. [x] Add eval fixtures for external-context alignment, not just diff-local correctness.
 92. [x] Add eval fixtures for merge-readiness judgments and unresolved-blocker classification.
 93. [x] Add eval fixtures for addressed-vs-stale finding lifecycle inference.
 94. [x] Add eval fixtures for multi-hop graph reasoning across call chains and contract edges.

eval/fixtures/repo_regressions/api_contract_breaking_status.yml

@@ -0,0 +1,24 @@
+name: repo regression - API status breaks documented contract
+repo_path: api_contract_repo
+diff: |
+  diff --git a/handlers.py b/handlers.py
+  index 1111111..2222222 100644
+  --- a/handlers.py
+  +++ b/handlers.py
+  @@ -1,3 +1,3 @@
+   def serialize_order(order):
+  -    status = "shipped" if order.shipped_at else "pending"
+  +    status = "fulfilled" if order.shipped_at else "pending"
+       return {"id": order.id, "status": status}
+expect:
+  must_find:
+    - file: handlers.py
+      contains_any:
+        - API contract drift
+        - openapi schema
+        - fulfilled is not part of the documented status enum
+        - documented enum only allows pending shipped or cancelled
+  must_not_find:
+    - contains: style
+  min_total: 1
+  max_total: 8

eval/fixtures/repo_regressions/api_contract_repo/handlers.py

@@ -0,0 +1,3 @@
+def serialize_order(order):
+    status = "shipped" if order.shipped_at else "pending"
+    return {"id": order.id, "status": status}

eval/fixtures/repo_regressions/api_contract_repo/openapi.yaml

@@ -0,0 +1,17 @@
+openapi: 3.1.0
+info:
+  title: Orders API
+  version: 1.0.0
+components:
+  schemas:
+    Order:
+      type: object
+      properties:
+        id:
+          type: string
+        status:
+          type: string
+          enum:
+            - pending
+            - shipped
+            - cancelled

eval/fixtures/repo_regressions/deploy_env_repo/charts/diffscope/values.yaml

@@ -0,0 +1,6 @@
+env:
+  - name: AUTOMATION_WEBHOOK_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: diffscope-secrets
+        key: automationWebhookSecret

eval/fixtures/repo_regressions/deploy_env_repo/settings.ts

@@ -0,0 +1,3 @@
+export function webhookSecret(env: NodeJS.ProcessEnv): string | undefined {
+  return env.AUTOMATION_WEBHOOK_SECRET
+}

eval/fixtures/repo_regressions/deployment_env_var_drift.yml

@@ -0,0 +1,24 @@
+name: repo regression - deployment env var rename not wired through charts
+repo_path: deploy_env_repo
+diff: |
+  diff --git a/settings.ts b/settings.ts
+  index 1111111..2222222 100644
+  --- a/settings.ts
+  +++ b/settings.ts
+  @@ -1,3 +1,3 @@
+   export function webhookSecret(env: NodeJS.ProcessEnv): string | undefined {
+  -  return env.AUTOMATION_WEBHOOK_SECRET
+  +  return env.REVIEW_WEBHOOK_SECRET
+   }
+expect:
+  must_find:
+    - file: settings.ts
+      contains_any:
+        - deployment configuration drift
+        - Helm chart still injects AUTOMATION_WEBHOOK_SECRET
+        - renamed environment variable is not wired through deployment manifests
+        - update charts or keep the old environment variable name
+  must_not_find:
+    - contains: style
+  min_total: 1
+  max_total: 8

eval/fixtures/repo_regressions/migration_drift_repo/migrations/0001_create_review_runs.sql

@@ -0,0 +1,4 @@
+create table review_runs (
+    id text primary key,
+    status text not null
+);

eval/fixtures/repo_regressions/migration_drift_repo/user_repo.rs

@@ -0,0 +1,6 @@
+pub fn insert_review(db: &Db, review: &Review) {
+    db.execute(
+        "insert into review_runs (id, status) values ($1, $2)",
+        &[&review.id, &review.status],
+    );
+}

eval/fixtures/repo_regressions/missing_review_run_migration.yml

@@ -0,0 +1,28 @@
+name: repo regression - review run column added without migration
+repo_path: migration_drift_repo
+diff: |
+  diff --git a/user_repo.rs b/user_repo.rs
+  index 1111111..2222222 100644
+  --- a/user_repo.rs
+  +++ b/user_repo.rs
+  @@ -1,6 +1,6 @@
+   pub fn insert_review(db: &Db, review: &Review) {
+       db.execute(
+  -        "insert into review_runs (id, status) values ($1, $2)",
+  +        "insert into review_runs (id, status, last_reviewed_at) values ($1, $2, $3)",
+  -        &[&review.id, &review.status],
+  +        &[&review.id, &review.status, &review.last_reviewed_at],
+       );
+   }
+expect:
+  must_find:
+    - file: user_repo.rs
+      contains_any:
+        - missing migration
+        - schema drift
+        - review_runs table does not define last_reviewed_at
+        - add a database migration for the new column
+  must_not_find:
+    - contains: style
+  min_total: 1
+  max_total: 8