fix: normalize repo names in oracle_checks.py to handle github.com/ prefix

MCP agents receive repo names with github.com/ prefix from Sourcegraph
tools. Haiku failed to strip this prefix in config-trace-010, causing
a false 0.0 score despite correct answer content. Added _normalize_repo()
that strips github.com/, gitlab.com/, bitbucket.org/ prefixes before
comparison in check_file_set_match, check_symbol_resolution, and
check_dependency_chain. Also adds /run-benchmark slash command.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sjarmak

.claude/commands/run-benchmark.md

@@ -0,0 +1,40 @@
+Configure and launch CodeContextBench benchmark runs.
+
+## Approval Gate (Required)
+
+Before executing any run, confirm with the user:
+
+1. **Model** — e.g. `anthropic/claude-haiku-4-5-20251001` (test) or `anthropic/claude-sonnet-4-6-20250514`
+2. **Suite / selection file** — which benchmark suite or `--selection-file`?
+3. **Config** — paired (default), `--baseline-only`, or `--full-only`? Which `--full-config`?
+4. **Parallel slots** — how many? (default: 1; use 8 for multi-account)
+5. **Category** — `staging` (default) or `official`?
+
+Do NOT launch until all five are confirmed.
+
+## Steps
+
+1. Run preflight checks:
+```bash
+python3 scripts/check_infra.py
+```
+
+2. Launch the run with confirmed parameters:
+```bash
+# MCP-unique tasks (artifact config)
+FULL_CONFIG=mcp-remote-artifact bash configs/run_selected_tasks.sh \
+  --selection-file configs/selected_mcp_unique_tasks.json \
+  --model <MODEL> --parallel <N> --category <CATEGORY>
+
+# SDLC suite
+./configs/<suite>_2config.sh --parallel <N>
+```
+
+3. Monitor progress:
+```bash
+python3 scripts/aggregate_status.py --staging
+```
+
+## Arguments
+
+$ARGUMENTS — optional: suite name, model, or selection file to pre-fill the approval gate

benchmarks/ccb_mcp_crossorg/ccx-crossorg-061/tests/oracle_checks.py

@@ -24,8 +24,24 @@
 import subprocess
 import sys
 from pathlib import Path
+import re
 from typing import Any, Dict, List, Optional
 
+# Hosting prefixes that Sourcegraph MCP tools prepend to repo names.
+# Agents sometimes forget to strip these; normalize before comparison.
+_HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
+
+
+def _normalize_repo(repo: str) -> str:
+    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+
+    >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    """
+    return _HOSTING_PREFIX_RE.sub("", repo)
+
 
 def check_file_set_match(
     answer_files: List[Dict[str, str]],
@@ -50,7 +66,7 @@ def check_file_set_match(
     [{'repo': 'a/b', 'path': 'x.go'}]
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""))
 
     oracle_set = {_key(f) for f in oracle_files}
     answer_set = {_key(f) for f in answer_files}
@@ -90,7 +106,7 @@ def check_symbol_resolution(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_set = {_key(s) for s in oracle_symbols}
     answer_set = {_key(s) for s in answer_symbols}
@@ -133,7 +149,7 @@ def check_dependency_chain(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_keys = [_key(s) for s in oracle_chain]
     answer_keys = [_key(s) for s in answer_chain]

benchmarks/ccb_mcp_crossorg/ccx-crossorg-066/tests/oracle_checks.py

@@ -24,8 +24,24 @@
 import subprocess
 import sys
 from pathlib import Path
+import re
 from typing import Any, Dict, List, Optional
 
+# Hosting prefixes that Sourcegraph MCP tools prepend to repo names.
+# Agents sometimes forget to strip these; normalize before comparison.
+_HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
+
+
+def _normalize_repo(repo: str) -> str:
+    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+
+    >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    """
+    return _HOSTING_PREFIX_RE.sub("", repo)
+
 
 def check_file_set_match(
     answer_files: List[Dict[str, str]],
@@ -50,7 +66,7 @@ def check_file_set_match(
     [{'repo': 'a/b', 'path': 'x.go'}]
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""))
 
     oracle_set = {_key(f) for f in oracle_files}
     answer_set = {_key(f) for f in answer_files}
@@ -90,7 +106,7 @@ def check_symbol_resolution(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_set = {_key(s) for s in oracle_symbols}
     answer_set = {_key(s) for s in answer_symbols}
@@ -133,7 +149,7 @@ def check_dependency_chain(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_keys = [_key(s) for s in oracle_chain]
     answer_keys = [_key(s) for s in answer_chain]

benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/tests/oracle_checks.py

@@ -24,8 +24,24 @@
 import subprocess
 import sys
 from pathlib import Path
+import re
 from typing import Any, Dict, List, Optional
 
+# Hosting prefixes that Sourcegraph MCP tools prepend to repo names.
+# Agents sometimes forget to strip these; normalize before comparison.
+_HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
+
+
+def _normalize_repo(repo: str) -> str:
+    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+
+    >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    """
+    return _HOSTING_PREFIX_RE.sub("", repo)
+
 
 def check_file_set_match(
     answer_files: List[Dict[str, str]],
@@ -50,7 +66,7 @@ def check_file_set_match(
     [{'repo': 'a/b', 'path': 'x.go'}]
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""))
 
     oracle_set = {_key(f) for f in oracle_files}
     answer_set = {_key(f) for f in answer_files}
@@ -90,7 +106,7 @@ def check_symbol_resolution(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_set = {_key(s) for s in oracle_symbols}
     answer_set = {_key(s) for s in answer_symbols}
@@ -133,7 +149,7 @@ def check_dependency_chain(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_keys = [_key(s) for s in oracle_chain]
     answer_keys = [_key(s) for s in answer_chain]

benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/tests/oracle_checks.py

@@ -24,8 +24,24 @@
 import subprocess
 import sys
 from pathlib import Path
+import re
 from typing import Any, Dict, List, Optional
 
+# Hosting prefixes that Sourcegraph MCP tools prepend to repo names.
+# Agents sometimes forget to strip these; normalize before comparison.
+_HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
+
+
+def _normalize_repo(repo: str) -> str:
+    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+
+    >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    """
+    return _HOSTING_PREFIX_RE.sub("", repo)
+
 
 def check_file_set_match(
     answer_files: List[Dict[str, str]],
@@ -50,7 +66,7 @@ def check_file_set_match(
     [{'repo': 'a/b', 'path': 'x.go'}]
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""))
 
     oracle_set = {_key(f) for f in oracle_files}
     answer_set = {_key(f) for f in answer_files}
@@ -90,7 +106,7 @@ def check_symbol_resolution(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_set = {_key(s) for s in oracle_symbols}
     answer_set = {_key(s) for s in answer_symbols}
@@ -133,7 +149,7 @@ def check_dependency_chain(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_keys = [_key(s) for s in oracle_chain]
     answer_keys = [_key(s) for s in answer_chain]

benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/tests/oracle_checks.py

@@ -24,8 +24,24 @@
 import subprocess
 import sys
 from pathlib import Path
+import re
 from typing import Any, Dict, List, Optional
 
+# Hosting prefixes that Sourcegraph MCP tools prepend to repo names.
+# Agents sometimes forget to strip these; normalize before comparison.
+_HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
+
+
+def _normalize_repo(repo: str) -> str:
+    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+
+    >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    """
+    return _HOSTING_PREFIX_RE.sub("", repo)
+
 
 def check_file_set_match(
     answer_files: List[Dict[str, str]],
@@ -50,7 +66,7 @@ def check_file_set_match(
     [{'repo': 'a/b', 'path': 'x.go'}]
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""))
 
     oracle_set = {_key(f) for f in oracle_files}
     answer_set = {_key(f) for f in answer_files}
@@ -90,7 +106,7 @@ def check_symbol_resolution(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_set = {_key(s) for s in oracle_symbols}
     answer_set = {_key(s) for s in answer_symbols}
@@ -133,7 +149,7 @@ def check_dependency_chain(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_keys = [_key(s) for s in oracle_chain]
     answer_keys = [_key(s) for s in answer_chain]

benchmarks/ccb_mcp_incident/ccx-incident-031/tests/oracle_checks.py

@@ -24,8 +24,24 @@
 import subprocess
 import sys
 from pathlib import Path
+import re
 from typing import Any, Dict, List, Optional
 
+# Hosting prefixes that Sourcegraph MCP tools prepend to repo names.
+# Agents sometimes forget to strip these; normalize before comparison.
+_HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
+
+
+def _normalize_repo(repo: str) -> str:
+    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+
+    >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
+    'sg-benchmarks/kubernetes-client-go'
+    """
+    return _HOSTING_PREFIX_RE.sub("", repo)
+
 
 def check_file_set_match(
     answer_files: List[Dict[str, str]],
@@ -50,7 +66,7 @@ def check_file_set_match(
     [{'repo': 'a/b', 'path': 'x.go'}]
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""))
 
     oracle_set = {_key(f) for f in oracle_files}
     answer_set = {_key(f) for f in answer_files}
@@ -90,7 +106,7 @@ def check_symbol_resolution(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_set = {_key(s) for s in oracle_symbols}
     answer_set = {_key(s) for s in answer_symbols}
@@ -133,7 +149,7 @@ def check_dependency_chain(
     1.0
     """
     def _key(item: Dict[str, str]) -> tuple:
-        return (item.get("repo", ""), item.get("path", ""), item.get("symbol", ""))
+        return (_normalize_repo(item.get("repo", "")), item.get("path", ""), item.get("symbol", ""))
 
     oracle_keys = [_key(s) for s in oracle_chain]
     answer_keys = [_key(s) for s in answer_chain]