|
37 | 37 | OFFICIAL_DIR = PROJECT_ROOT / "runs" / "official" |
38 | 38 | VALIDATE_SCRIPT = PROJECT_ROOT / "scripts" / "validate_task_run.py" |
39 | 39 | MANIFEST_SCRIPT = PROJECT_ROOT / "scripts" / "generate_manifest.py" |
| 40 | +EXTRACT_METRICS_SCRIPT = PROJECT_ROOT / "scripts" / "extract_task_metrics.py" |
| 41 | +SELECTED_TASKS_FILE = PROJECT_ROOT / "configs" / "selected_benchmark_tasks.json" |
40 | 42 |
|
41 | 43 | SKIP_PATTERNS = ["__broken_verifier", "validation_test", "archive", "__v1_hinted"] |
42 | 44 | CONFIGS = ["baseline", "sourcegraph_base", "sourcegraph_full", "sourcegraph_isolated", "sourcegraph_only"] |
43 | 45 |
|
| 46 | +DIR_PREFIX_TO_SUITE = { |
| 47 | + "bigcode_mcp_": "ccb_largerepo", |
| 48 | + "bigcode_sgcompare_": "ccb_largerepo", |
| 49 | + "codereview_": "ccb_codereview", |
| 50 | + "crossrepo_": "ccb_crossrepo", |
| 51 | + "dependeval_": "ccb_dependeval", |
| 52 | + "dibench_": "ccb_dibench", |
| 53 | + "docgen_": "ccb_docgen", |
| 54 | + "enterprise_": "ccb_enterprise", |
| 55 | + "governance_": "ccb_governance", |
| 56 | + "investigation_": "ccb_investigation", |
| 57 | + "k8s_docs_": "ccb_k8sdocs", |
| 58 | + "linuxflbench_": "ccb_linuxflbench", |
| 59 | + "locobench_": "ccb_locobench", |
| 60 | + "navprove_": "ccb_navprove", |
| 61 | + "nlqa_": "ccb_nlqa", |
| 62 | + "onboarding_": "ccb_onboarding", |
| 63 | + "paired_rerun_crossrepo_": "ccb_crossrepo", |
| 64 | + "paired_rerun_dibench_": "ccb_dibench", |
| 65 | + "paired_rerun_pytorch_": "ccb_pytorch", |
| 66 | + "pytorch_": "ccb_pytorch", |
| 67 | + "repoqa_": "ccb_repoqa", |
| 68 | + "security_": "ccb_security", |
| 69 | + "swebenchpro_": "ccb_swebenchpro", |
| 70 | + "sweperf_": "ccb_sweperf", |
| 71 | + "tac_": "ccb_tac", |
| 72 | +} |
| 73 | + |
44 | 74 | # ANSI colors |
45 | 75 | GREEN = "\033[92m" |
46 | 76 | YELLOW = "\033[93m" |
@@ -94,6 +124,120 @@ def find_task_dirs(config_path: Path) -> list[Path]: |
94 | 124 | return task_dirs |
95 | 125 |
|
96 | 126 |
|
| 127 | +def suite_from_task_id(task_id: str) -> str | None: |
| 128 | + """Infer suite from task_id when run name isn't enough.""" |
| 129 | + if task_id.startswith("instance_"): |
| 130 | + return "ccb_swebenchpro" |
| 131 | + if task_id.startswith("sgt-"): |
| 132 | + return "ccb_pytorch" |
| 133 | + if task_id.startswith("big-code-"): |
| 134 | + return "ccb_largerepo" |
| 135 | + if task_id.startswith("dibench-"): |
| 136 | + return "ccb_dibench" |
| 137 | + if task_id.startswith("cr-"): |
| 138 | + return "ccb_codereview" |
| 139 | + if task_id.endswith("-doc-001"): |
| 140 | + return "ccb_k8sdocs" |
| 141 | + if task_id.startswith("lfl-"): |
| 142 | + return "ccb_linuxflbench" |
| 143 | + if task_id.startswith("bug_localization_") or task_id.startswith("refactor_rename_") or task_id.startswith("cross_file_reasoning_"): |
| 144 | + return "ccb_crossrepo" |
| 145 | + if "_expert_" in task_id: |
| 146 | + return "ccb_locobench" |
| 147 | + if task_id.startswith("multifile_editing-") or task_id.startswith("file_span_fix-") or task_id.startswith("dependency_recognition-"): |
| 148 | + return "ccb_dependeval" |
| 149 | + if task_id.startswith("repoqa-"): |
| 150 | + return "ccb_repoqa" |
| 151 | + if task_id.startswith("sweperf-") or task_id.startswith("sweperf_") or task_id.startswith("django_perf_"): |
| 152 | + return "ccb_sweperf" |
| 153 | + if task_id.startswith("tac-") or task_id.startswith("simple_test_") or task_id.startswith("api_upgrade_") or task_id.startswith("hyperloglog") or task_id.startswith("write-unit-test"): |
| 154 | + return "ccb_tac" |
| 155 | + if "answer_extraction" in task_id or "function_recall" in task_id or "question_answer" in task_id: |
| 156 | + return "ccb_repoqa" |
| 157 | + if task_id.startswith("onboard-"): |
| 158 | + return "ccb_onboarding" |
| 159 | + if task_id.startswith("gov-"): |
| 160 | + return "ccb_governance" |
| 161 | + if task_id.startswith("sec-"): |
| 162 | + return "ccb_security" |
| 163 | + if task_id.startswith("ent-") or task_id.startswith("dep-") or task_id.startswith("multi-team-") or task_id.startswith("polyglot-"): |
| 164 | + return "ccb_enterprise" |
| 165 | + return None |
| 166 | + |
| 167 | + |
| 168 | +def suite_from_run_name(run_name: str) -> str | None: |
| 169 | + for prefix, suite in DIR_PREFIX_TO_SUITE.items(): |
| 170 | + if run_name.startswith(prefix): |
| 171 | + return suite |
| 172 | + return None |
| 173 | + |
| 174 | + |
| 175 | +def benchmark_for_task(run_name: str, task_dir: Path) -> str | None: |
| 176 | + suite = suite_from_run_name(run_name) |
| 177 | + if suite: |
| 178 | + return suite |
| 179 | + task_name = task_dir.name.rsplit("__", 1)[0] |
| 180 | + return suite_from_task_id(task_name) |
| 181 | + |
| 182 | + |
| 183 | +def extract_missing_task_metrics(run_dir: Path, *, execute: bool) -> tuple[int, int, list[str]]: |
| 184 | + """Generate missing task_metrics.json files for a run. |
| 185 | +
|
| 186 | + Returns: |
| 187 | + (missing_count, generated_count, errors) |
| 188 | + """ |
| 189 | + missing_count = 0 |
| 190 | + generated_count = 0 |
| 191 | + errors: list[str] = [] |
| 192 | + |
| 193 | + for config_name in CONFIGS: |
| 194 | + config_path = run_dir / config_name |
| 195 | + if not config_path.is_dir(): |
| 196 | + continue |
| 197 | + |
| 198 | + for task_dir in find_task_dirs(config_path): |
| 199 | + result_json = task_dir / "result.json" |
| 200 | + metrics_json = task_dir / "task_metrics.json" |
| 201 | + if not result_json.is_file() or metrics_json.is_file(): |
| 202 | + continue |
| 203 | + |
| 204 | + missing_count += 1 |
| 205 | + if not execute: |
| 206 | + continue |
| 207 | + |
| 208 | + benchmark = benchmark_for_task(run_dir.name, task_dir) |
| 209 | + if benchmark is None: |
| 210 | + errors.append(f"{config_name}/{task_dir.name}: could not infer benchmark") |
| 211 | + continue |
| 212 | + |
| 213 | + cmd = [ |
| 214 | + sys.executable, |
| 215 | + str(EXTRACT_METRICS_SCRIPT), |
| 216 | + "--task-dir", |
| 217 | + str(task_dir), |
| 218 | + "--benchmark", |
| 219 | + benchmark, |
| 220 | + "--config", |
| 221 | + config_name, |
| 222 | + ] |
| 223 | + if SELECTED_TASKS_FILE.is_file(): |
| 224 | + cmd.extend(["--selected-tasks", str(SELECTED_TASKS_FILE)]) |
| 225 | + |
| 226 | + proc = subprocess.run( |
| 227 | + cmd, |
| 228 | + capture_output=True, |
| 229 | + text=True, |
| 230 | + timeout=120, |
| 231 | + ) |
| 232 | + if proc.returncode == 0 and metrics_json.is_file(): |
| 233 | + generated_count += 1 |
| 234 | + else: |
| 235 | + detail = proc.stderr.strip() or proc.stdout.strip() or f"exit {proc.returncode}" |
| 236 | + errors.append(f"{config_name}/{task_dir.name}: {detail[:240]}") |
| 237 | + |
| 238 | + return missing_count, generated_count, errors |
| 239 | + |
| 240 | + |
97 | 241 | def validate_run(run_dir: Path) -> ValidationResult: |
98 | 242 | """Validate a staging run and return results.""" |
99 | 243 | result = ValidationResult(run_name=run_dir.name) |
@@ -336,6 +480,35 @@ def cmd_promote( |
336 | 480 | skipped.append(run_dir.name) |
337 | 481 | continue |
338 | 482 |
|
| 483 | + missing_metrics, generated_metrics, metric_errors = extract_missing_task_metrics( |
| 484 | + run_dir, execute=execute |
| 485 | + ) |
| 486 | + |
| 487 | + if execute: |
| 488 | + if missing_metrics == 0: |
| 489 | + print(f"\n {GREEN}Metrics:{RESET} all task_metrics.json files already present.") |
| 490 | + else: |
| 491 | + print( |
| 492 | + f"\n Metrics extraction: {generated_metrics}/{missing_metrics} generated" |
| 493 | + ) |
| 494 | + if metric_errors: |
| 495 | + print(f" {RED}Extraction errors:{RESET} {len(metric_errors)}") |
| 496 | + for err in metric_errors[:5]: |
| 497 | + print(f" - {err}") |
| 498 | + if len(metric_errors) > 5: |
| 499 | + print(f" ... {len(metric_errors) - 5} more") |
| 500 | + if not force: |
| 501 | + print( |
| 502 | + f"\n {RED}BLOCKED:{RESET} Failed to generate all task_metrics.json files." |
| 503 | + ) |
| 504 | + print(" Use --force to bypass.") |
| 505 | + skipped.append(run_dir.name) |
| 506 | + continue |
| 507 | + elif missing_metrics > 0: |
| 508 | + print( |
| 509 | + f"\n {BOLD}DRY RUN:{RESET} would generate {missing_metrics} missing task_metrics.json file(s) before move." |
| 510 | + ) |
| 511 | + |
339 | 512 | if execute: |
340 | 513 | print(f"\n Promoting: {run_dir.name}") |
341 | 514 | shutil.move(str(run_dir), str(official_dest)) |
|
0 commit comments