From a12d078eeb981c4bbd18f58ab419383ed940a061 Mon Sep 17 00:00:00 2001 From: carlos-alm Date: Fri, 15 May 2026 01:59:44 -0600 Subject: [PATCH] perf(bench): exclude resolution fixtures from incremental-benchmark sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The incremental benchmark walks the repo root, which pulls hand-annotated resolution-benchmark fixtures into the corpus. A single heavy grammar (tree-sitter-verilog #1107) added ~850ms to fullBuildMs — the cost is real but unrelated to shared code paths the benchmark is meant to track. Add an optional exclude glob array to BuildGraphOpts, merged into config.exclude in setupPipeline so both the JS pipeline and the native orchestrator (which receives ctx.config as JSON) honor it. Pass 'tests/benchmarks/resolution/fixtures/**' from the benchmark script so future language PRs don't silently inflate timings, and remove the 3.10.0:Full build entry from KNOWN_REGRESSIONS now that the cause is addressed. Closes #1112 --- scripts/incremental-benchmark.ts | 21 +++++++++++++++------ src/domain/graph/builder/pipeline.ts | 3 +++ src/types.ts | 6 ++++++ tests/benchmarks/regression-guard.test.ts | 13 ------------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/scripts/incremental-benchmark.ts b/scripts/incremental-benchmark.ts index a37f132c..f3d19a7f 100644 --- a/scripts/incremental-benchmark.ts +++ b/scripts/incremental-benchmark.ts @@ -181,6 +181,15 @@ const PROBE_FILE = path.join(root, 'src', 'domain', 'queries.ts'); // CI-amplified false regressions on sub-30ms metrics like No-op rebuild. const WARMUP_RUNS = 2; +// Resolution-benchmark fixtures live under the repo root and get pulled into +// every self-build sweep. They are hand-annotated test corpora — not real +// codegraph code — and a single heavy grammar (e.g. tree-sitter-verilog) can +// add hundreds of milliseconds to fullBuildMs purely from fixture parsing +// (#1112). Exclude them so adding native support for a new language doesn't +// silently inflate the incremental-benchmark numbers. +const BENCH_EXCLUDE = ['tests/benchmarks/resolution/fixtures/**']; +const BUILD_OPTS = { engine, exclude: BENCH_EXCLUDE }; + function median(arr) { const sorted = [...arr].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); @@ -194,7 +203,7 @@ const fullTimings = []; for (let i = 0; i < RUNS; i++) { if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); const start = performance.now(); - await buildGraph(root, { engine, incremental: false }); + await buildGraph(root, { ...BUILD_OPTS, incremental: false }); fullTimings.push(performance.now() - start); } const fullBuildMs = Math.round(median(fullTimings)); @@ -203,12 +212,12 @@ const fullBuildMs = Math.round(median(fullTimings)); let noopRebuildMs = null; try { for (let i = 0; i < WARMUP_RUNS; i++) { - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); } const noopTimings = []; for (let i = 0; i < RUNS; i++) { const start = performance.now(); - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); noopTimings.push(performance.now() - start); } noopRebuildMs = Math.round(median(noopTimings)); @@ -223,13 +232,13 @@ let oneFilePhases = null; try { for (let i = 0; i < WARMUP_RUNS; i++) { fs.writeFileSync(PROBE_FILE, original + `\n// warmup-${i}\n`); - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); } const oneFileRuns = []; for (let i = 0; i < RUNS; i++) { fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); const start = performance.now(); - const res = await buildGraph(root, { engine, incremental: true }); + const res = await buildGraph(root, { ...BUILD_OPTS, incremental: true }); oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null }); } oneFileRuns.sort((a, b) => a.ms - b.ms); @@ -241,7 +250,7 @@ try { } finally { fs.writeFileSync(PROBE_FILE, original); try { - await buildGraph(root, { engine, incremental: true }); + await buildGraph(root, { ...BUILD_OPTS, incremental: true }); } catch { // Cleanup rebuild failed — probe file is already restored, move on } diff --git a/src/domain/graph/builder/pipeline.ts b/src/domain/graph/builder/pipeline.ts index f31afa77..1d67f4d2 100644 --- a/src/domain/graph/builder/pipeline.ts +++ b/src/domain/graph/builder/pipeline.ts @@ -183,6 +183,9 @@ function setupPipeline(ctx: PipelineContext): void { initSchema(ctx.db); ctx.config = loadConfig(ctx.rootDir); + if (ctx.opts.exclude && ctx.opts.exclude.length > 0) { + ctx.config = { ...ctx.config, exclude: [...ctx.config.exclude, ...ctx.opts.exclude] }; + } ctx.incremental = ctx.opts.incremental !== false && ctx.config.build && ctx.config.build.incremental !== false; diff --git a/src/types.ts b/src/types.ts index 08eb9431..6d8f31b7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1064,6 +1064,12 @@ export interface BuildGraphOpts { cfg?: boolean; scope?: string[]; skipRegistry?: boolean; + /** + * Extra exclude globs appended to `config.exclude`. Lets benchmark scripts + * skip fixture directories that bloat self-build timings without permanently + * affecting `.codegraphrc.json` for normal users. + */ + exclude?: string[]; } /** Build timing result from buildGraph. */ diff --git a/tests/benchmarks/regression-guard.test.ts b/tests/benchmarks/regression-guard.test.ts index 4125325e..1694c0d5 100644 --- a/tests/benchmarks/regression-guard.test.ts +++ b/tests/benchmarks/regression-guard.test.ts @@ -176,18 +176,6 @@ const SKIP_VERSIONS = new Set(['3.8.0']); * absolute delta 10.4ms exactly at the MIN_ABSOLUTE_DELTA floor. Exempt * this release; remove once 3.11.0+ data confirms stabilization. * - * - 3.10.0:Full build — adding native Verilog support (#1107) pulled the - * 4 `.v` resolution-benchmark fixtures into the corpus the incremental - * benchmark sweeps (it runs against the repo root). tree-sitter-verilog - * is a large grammar (SystemVerilog is one of the heaviest in the - * tree-sitter ecosystem) so each file costs noticeably more than the - * other fixture languages. Local measurement: 1959 → 2809 (+43%, run - * 25716010487). The cost is real and structural — not a regression in - * shared code paths. Resolution: either exclude `tests/benchmarks/ - * resolution/fixtures/verilog/**` from the benchmark sweep or accept the - * one-time bump as the cost of supporting Verilog. Tracked separately; - * exempt this release. - * * - 3.10.0:Query time — cumulative effect of adding two native extractors * (Solidity #1100 + R #1102) in quick succession. Neither tripped the * threshold individually (Solidity PR's Query time stayed at 49ms, R PR @@ -230,7 +218,6 @@ const KNOWN_REGRESSIONS = new Set([ '3.10.0:fnDeps depth 1', '3.10.0:fnDeps depth 3', '3.10.0:fnDeps depth 5', - '3.10.0:Full build', '3.10.0:Query time', ]);