fix: add normalize_repo to 3 design F1 scorers + fix Daytona disk limit

sjarmak · claude · sjarmak · commit 4d522d3e7003 · 2026-03-01T02:13:36.000Z
- Add normalize_repo() to make_key() in 3 ccb_design F1 scorer test.sh
  files (envoy-stream-aggregated-sym, k8s-sharedinformer-sym,
  terraform-provider-iface-sym) so sg-evals mirror names match upstream
  repo names in ground truth
- Fix daytona_runner.py disk floor from 20GB to 10GB to fit Daytona
  Tier 3 sandbox limit

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/ccb_design/envoy-stream-aggregated-sym-001/tests/test.sh b/benchmarks/ccb_design/envoy-stream-aggregated-sym-001/tests/test.sh
@@ -98,10 +98,44 @@ try:
         write_reward(0.0)
         sys.exit(0)
 
+    # ── Repo name normalization ────────────────────────────────────────────
+    def normalize_repo(name):
+        """Normalize repo names to a canonical short form for matching.
+
+        Handles sg-evals mirror names and upstream org/repo names by
+        extracting just the repo basename (last path segment).
+
+        sg-evals/envoy--2300e245       -> envoy
+        github.com/sg-evals/envoy--h   -> envoy
+        envoyproxy/envoy               -> envoy
+        grpc/grpc-go                   -> grpc-go
+        """
+        n = name.strip()
+        # Strip URL prefixes
+        for prefix in ("github.com/", "https://github.com/"):
+            if n.startswith(prefix):
+                n = n[len(prefix):]
+        # Strip sg-evals/ prefix
+        if n.startswith("sg-evals/"):
+            n = n[len("sg-evals/"):]
+        # Strip --hexhash suffix (8+ hex chars after --)
+        n = re.sub(r'--[0-9a-f]{7,}$', '', n)
+        # Take just the last path segment (repo basename)
+        if "/" in n:
+            n = n.rsplit("/", 1)[-1]
+        return n
+
     # ── Build composite keys ─────────────────────────────────────────────
     def make_key(entry, fields):
-        """Build a composite key tuple from an entry's field values."""
-        return tuple(str(entry.get(f, "")).strip() for f in fields)
+        """Build a composite key tuple from an entry's field values.
+        Normalizes 'repo' field to handle sg-evals mirror names."""
+        parts = []
+        for f in fields:
+            val = str(entry.get(f, "")).strip()
+            if f == "repo":
+                val = normalize_repo(val)
+            parts.append(val)
+        return tuple(parts)
 
     expected_keys = [make_key(e, key_fields) for e in expected]
     reported_keys = [make_key(r, key_fields) for r in reported]
diff --git a/benchmarks/ccb_design/k8s-sharedinformer-sym-001/tests/test.sh b/benchmarks/ccb_design/k8s-sharedinformer-sym-001/tests/test.sh
@@ -99,10 +99,44 @@ try:
         write_reward(0.0)
         sys.exit(0)
 
+    # ── Repo name normalization ────────────────────────────────────────────
+    def normalize_repo(name):
+        """Normalize repo names to a canonical short form for matching.
+
+        Handles sg-evals mirror names and upstream org/repo names by
+        extracting just the repo basename (last path segment).
+
+        sg-evals/kubernetes--abc123    -> kubernetes
+        github.com/sg-evals/k8s--h    -> k8s
+        kubernetes/kubernetes          -> kubernetes
+        kubernetes/autoscaler          -> autoscaler
+        """
+        n = name.strip()
+        # Strip URL prefixes
+        for prefix in ("github.com/", "https://github.com/"):
+            if n.startswith(prefix):
+                n = n[len(prefix):]
+        # Strip sg-evals/ prefix
+        if n.startswith("sg-evals/"):
+            n = n[len("sg-evals/"):]
+        # Strip --hexhash suffix (8+ hex chars after --)
+        n = re.sub(r'--[0-9a-f]{7,}$', '', n)
+        # Take just the last path segment (repo basename)
+        if "/" in n:
+            n = n.rsplit("/", 1)[-1]
+        return n
+
     # ── Build composite keys ─────────────────────────────────────────────
     def make_key(entry, fields):
-        """Build a composite key tuple from an entry's field values."""
-        return tuple(str(entry.get(f, "")).strip() for f in fields)
+        """Build a composite key tuple from an entry's field values.
+        Normalizes 'repo' field to handle sg-evals mirror names."""
+        parts = []
+        for f in fields:
+            val = str(entry.get(f, "")).strip()
+            if f == "repo":
+                val = normalize_repo(val)
+            parts.append(val)
+        return tuple(parts)
 
     expected_keys = [make_key(e, key_fields) for e in expected]
     reported_keys = [make_key(r, key_fields) for r in reported]
diff --git a/benchmarks/ccb_design/terraform-provider-iface-sym-001/tests/test.sh b/benchmarks/ccb_design/terraform-provider-iface-sym-001/tests/test.sh
@@ -99,10 +99,44 @@ try:
         write_reward(0.0)
         sys.exit(0)
 
+    # ── Repo name normalization ────────────────────────────────────────────
+    def normalize_repo(name):
+        """Normalize repo names to a canonical short form for matching.
+
+        Handles sg-evals mirror names and upstream org/repo names by
+        extracting just the repo basename (last path segment).
+
+        sg-evals/terraform--abc123     -> terraform
+        github.com/sg-evals/tf--h     -> tf
+        hashicorp/terraform            -> terraform
+        hashicorp/terraform-provider-aws -> terraform-provider-aws
+        """
+        n = name.strip()
+        # Strip URL prefixes
+        for prefix in ("github.com/", "https://github.com/"):
+            if n.startswith(prefix):
+                n = n[len(prefix):]
+        # Strip sg-evals/ prefix
+        if n.startswith("sg-evals/"):
+            n = n[len("sg-evals/"):]
+        # Strip --hexhash suffix (8+ hex chars after --)
+        n = re.sub(r'--[0-9a-f]{7,}$', '', n)
+        # Take just the last path segment (repo basename)
+        if "/" in n:
+            n = n.rsplit("/", 1)[-1]
+        return n
+
     # ── Build composite keys ─────────────────────────────────────────────
     def make_key(entry, fields):
-        """Build a composite key tuple from an entry's field values."""
-        return tuple(str(entry.get(f, "")).strip() for f in fields)
+        """Build a composite key tuple from an entry's field values.
+        Normalizes 'repo' field to handle sg-evals mirror names."""
+        parts = []
+        for f in fields:
+            val = str(entry.get(f, "")).strip()
+            if f == "repo":
+                val = normalize_repo(val)
+            parts.append(val)
+        return tuple(parts)
 
     expected_keys = [make_key(e, key_fields) for e in expected]
     reported_keys = [make_key(r, key_fields) for r in reported]
diff --git a/scripts/daytona_runner.py b/scripts/daytona_runner.py
@@ -400,7 +400,7 @@ def create_sandbox(self, task: TaskSpec):
             resources=Resources(
                 cpu=task.cpus,
                 memory=max(task.memory_mb // 1024, 1),
-                disk=max(task.storage_mb // 1024, 20),
+                disk=max(task.storage_mb // 1024, 10),
             ),
             auto_stop_interval=0,
         )

Original file line number	Diff line number	Diff line change
`@@ -400,7 +400,7 @@ def create_sandbox(self, task: TaskSpec):`
`400`	`400`	`resources=Resources(`
`401`	`401`	`cpu=task.cpus,`
`402`	`402`	`memory=max(task.memory_mb // 1024, 1),`
`403`		`- disk=max(task.storage_mb // 1024, 20),`
	`403`	`+ disk=max(task.storage_mb // 1024, 10),`
`404`	`404`	`),`
`405`	`405`	`auto_stop_interval=0,`
`406`	`406`	`)`