fix: oracle scoring — strip github.com/ prefix, match version-tag mirror suffixes

Two bugs caused 73/211 MCP-unique tasks to score 0.0 regardless of agent quality:

1. _normalize_file_entry (hydrate_task_specs.py) and _coerce_file_entry
   (oracle_checks.py): entries like "github.com/sg-evals/kubernetes--v1.32.0/pkg/file.go"
   were split as repo="github.com/sg-evals", path="kubernetes--v1.32.0/pkg/file.go".
   Fix: strip "github.com/" prefix before splitting, yielding
   repo="sg-evals/kubernetes--v1.32.0", path="pkg/file.go".

2. _MIRROR_HASH_RE only matched hex hashes (--871325b8) not version tags
   (--v1.32.0). Fix: broaden regex to --(v[\d.]+|[0-9a-f]{6,})$ so
   kubernetes--v1.32.0 normalizes to "kubernetes" and matches agent's
   "kubernetes/kubernetes".

Re-hydrated all 211 task_spec.json + copied oracle_checks.py to 211 task dirs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sjarmak

benchmarks/ccb_mcp_compliance/ccx-compliance-051/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-052/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-053/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-057/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-115/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-118/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-124/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-182/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-183/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}
 
 

benchmarks/ccb_mcp_compliance/ccx-compliance-184/tests/oracle_checks.py

@@ -35,24 +35,30 @@
 # upstream names like "mozilla-firefox/firefox" or "openjdk/jdk".  Normalise
 # both sides so that matching works regardless of which convention is used.
 
-_MIRROR_HASH_RE = re.compile(r"--[0-9a-f]{6,}$")
+_MIRROR_HASH_RE = re.compile(r"--(v[\d.]+|[0-9a-f]{6,})$")
 
 
 def _coerce_file_entry(entry) -> Dict[str, str]:
     """Coerce a file entry to {"repo": ..., "path": ...} dict format.
 
     Handles string entries like "sg-evals/kubernetes--v1.32.0/pkg/file.go"
     where the first two path components are the repo.
+
+    Also handles "github.com/sg-evals/repo--hash/path" by stripping the
+    "github.com/" prefix first so the repo is "sg-evals/repo--hash".
     """
     if isinstance(entry, dict):
         return entry
     if isinstance(entry, str):
-        parts = entry.split("/", 2)
+        s = entry
+        if s.startswith("github.com/"):
+            s = s[len("github.com/"):]
+        parts = s.split("/", 2)
         if len(parts) >= 3:
             return {"repo": f"{parts[0]}/{parts[1]}", "path": parts[2]}
         elif len(parts) == 2:
             return {"repo": parts[0], "path": parts[1]}
-        return {"repo": "", "path": entry}
+        return {"repo": "", "path": s}
     return {"repo": "", "path": str(entry)}