diff --git a/CHANGELOG.md b/CHANGELOG.md index 09fbad95a..2999ea1f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,14 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph init` now builds the initial index by default — you no longer need the `-i`/`--index` flag (it's still accepted, so existing commands and scripts keep working). (#483) - Go: Gin middleware chains now connect end-to-end in `codegraph_trace` and `codegraph_explore` — following a request reaches the middleware and route handlers registered via `.Use()` / `.GET()` instead of dead-ending where the framework dispatches the chain dynamically. - `codegraph_explore` now sizes its response to the *answer* instead of the file count: it shows the mechanism and the exact methods you asked about in full — even when they're buried deep in a large file — while collapsing the redundant interchangeable implementations of an interface (an HTTP interceptor chain, a query-compiler family) down to signatures. Fewer tokens for a more complete answer, so on the flows that used to occasionally cost more than plain grep/read it's now clearly cheaper — and the win holds across small, medium, and large codebases. Distinct, non-interchangeable code is shown in full as before. Disable with `CODEGRAPH_ADAPTIVE_EXPLORE=0`. +- Swift deferred-validation flows (and similar "handler array" patterns) now connect end-to-end in `codegraph_trace` and `codegraph_explore` — following a request's lifecycle reaches the validators registered with `.validate { … }` instead of dead-ending where the framework runs them by iterating a stored list of closures. Any pattern where closures are appended to a collection and later invoked by looping over it is now traced. +- `codegraph_explore` now spells out the dynamic-dispatch relationships of the symbols you ask about — e.g. "the closures registered here are run by `didCompleteTask`" — so the indirect hops you'd otherwise grep to reconstruct are listed alongside the call flow. +- `codegraph_explore` answers multi-phase questions that span a large "god file" far more completely. For a flow like "build, send, and validate a request" — where one big file holds the build chain and the validate logic lives in others — it now keeps every method *on the flow path* in full, collapses the file's off-path methods to one-line signatures, and guarantees each phase's defining file is shown (instead of truncating at a fixed size and dropping whichever phase came last, which sent you to read it by hand). Incidental files that merely name-drop the flow are still trimmed, so the response stays focused on the code that answers the question. ### Fixes +- `codegraph_trace` now resolves an overloaded symbol name to its real implementation instead of an empty protocol/delegate stub. Tracing a flow through a heavily-overloaded API (common in Swift, Java, C#, and Go) could land on an unrelated no-op method that happened to share the name and report "no path"; it now picks the substantive definition the flow actually runs through. +- CodeGraph's MCP server now answers an agent's opening handshake the instant it launches instead of blocking while the index loads, so a fresh session's very first tool call no longer occasionally races a server that's still warming up and falls back to grep/read. The first question in a new session now reliably goes through CodeGraph. - Indexing a project that contains only config-style files (YAML, Twig, or `.properties`) no longer misleadingly reports "No files found to index" — these files are tracked at the file level and are now counted as indexed. Thanks @luojiyin1987 (#357). ## [0.9.7] - 2026-05-28 diff --git a/__tests__/adaptive-explore-sizing.test.ts b/__tests__/adaptive-explore-sizing.test.ts index a1a531cc9..809a885cf 100644 --- a/__tests__/adaptive-explore-sizing.test.ts +++ b/__tests__/adaptive-explore-sizing.test.ts @@ -31,7 +31,9 @@ import * as os from 'os'; import { ToolHandler } from '../src/mcp/tools'; import CodeGraph from '../src/index'; -const SKELETON_MARK = '· skeleton (signatures only; Read for a full body)'; +// Stable marker — assert the `· skeleton` tag, not its exact trailing wording +// (the steer-to-explore phrasing changed when the Read invitation was removed). +const SKELETON_MARK = '· skeleton (signatures only'; /** Return the `#### ...` section for a file basename, header through the * line before the next `###`/`####` header (or end of output). */ diff --git a/__tests__/closure-collection-synthesizer.test.ts b/__tests__/closure-collection-synthesizer.test.ts new file mode 100644 index 000000000..b516cb496 --- /dev/null +++ b/__tests__/closure-collection-synthesizer.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { CodeGraph } from '../src'; + +/** + * End-to-end synthesizer test for closure-collection dynamic dispatch. + * + * A method appends a closure to a collection property; another method iterates + * that property *invoking each element* (`coll.forEach { $0() }`) — a dynamic + * dispatch tree-sitter can't resolve, so a flow into the dispatcher dead-ends + * before the registered closures. This is Alamofire's request-validation shape: + * `DataRequest.validate` does `validators.write { $0.append(validator) }`, the + * base `Request.didCompleteTask` runs `validators.forEach { $0() }`. + * + * Verify the synthesizer (1) links the dispatcher → each same-named registrar + * across files/classes, (2) handles both the Swift `prop.write { $0.append }` + * and the direct `prop.append(...)` registrar forms, (3) surfaces the wiring + * site, and (4) does NOT fire on a `.forEach` that doesn't invoke its element + * (the closure-invoke is the precision gate — a plain collection is skipped). + */ +describe('closure-collection synthesizer', () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'closure-coll-fixture-')); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('links dispatcher → registrars across files, both append forms, and skips non-invoked collections', async () => { + // Base class: the dispatchers (iterate-and-invoke) + a non-closure control. + fs.writeFileSync( + path.join(dir, 'Request.swift'), + `class Request { + var validators: [() -> Void] = [] + var handlers: [() -> Void] = [] + var names: [String] = [] + + func didCompleteTask() { + let validators = validators + validators.forEach { $0() } + } + + func runHandlers() { + handlers.forEach { $0() } + } + + func printNames() { + names.forEach { print($0) } + } +} +` + ); + + // Subclass: the registrars (append a closure) in a DIFFERENT file/class. + fs.writeFileSync( + path.join(dir, 'DataRequest.swift'), + `class DataRequest: Request { + func validate(_ validation: @escaping () -> Void) -> Self { + let validator: () -> Void = { validation() } + validators.write { $0.append(validator) } + return self + } + + func onEvent(_ handler: @escaping () -> Void) { + handlers.append(handler) + } + + func addName(_ n: String) { + names.append(n) + } +} +` + ); + + const cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + + const db = (cg as any).db.db; + const rows = db + .prepare( + `SELECT s.name source_name, s.kind source_kind, t.name target_name, + json_extract(e.metadata,'$.field') field, + json_extract(e.metadata,'$.registeredAt') registeredAt + FROM edges e + JOIN nodes s ON s.id = e.source + JOIN nodes t ON t.id = e.target + WHERE json_extract(e.metadata,'$.synthesizedBy') = 'closure-collection'` + ) + .all(); + cg.close?.(); + + expect(rows.length).toBeGreaterThan(0); + + // Every edge originates from a dispatcher method and is a real `calls` hop. + expect(rows.every((r: any) => r.source_kind === 'method')).toBe(true); + + // The validators flow: didCompleteTask → validate, captured via the Swift + // Protected `prop.write { $0.append }` form, wiring site surfaced. + const validatorsEdge = rows.find( + (r: any) => r.field === 'validators' && r.target_name === 'validate' + ); + expect(validatorsEdge).toBeTruthy(); + expect(validatorsEdge.source_name).toBe('didCompleteTask'); + expect(validatorsEdge.registeredAt).toMatch(/DataRequest\.swift:\d+/); + + // The handlers flow: runHandlers → onEvent, via the direct `prop.append` + // form — proves both registrar shapes are covered. + const handlersEdge = rows.find( + (r: any) => r.field === 'handlers' && r.target_name === 'onEvent' + ); + expect(handlersEdge).toBeTruthy(); + expect(handlersEdge.source_name).toBe('runHandlers'); + + // Precision gate: `names.forEach { print($0) }` does NOT invoke its element, + // so `names` is not a closure collection — no edge, and addName is never a target. + expect(rows.some((r: any) => r.field === 'names')).toBe(false); + expect(rows.some((r: any) => r.target_name === 'addName')).toBe(false); + }); +}); diff --git a/__tests__/mcp-daemon.test.ts b/__tests__/mcp-daemon.test.ts index fa1558f02..1711d673d 100644 --- a/__tests__/mcp-daemon.test.ts +++ b/__tests__/mcp-daemon.test.ts @@ -346,11 +346,12 @@ describe('Shared MCP daemon (issue #411)', () => { servers.push(server); sendInitialize(server.child, `file://${tempDir}`, 1); // Despite the mismatched daemon, the client still gets an initialize - // response — the proxy refuses to attach and falls back to direct mode. + // response — the proxy answers the handshake locally and, refusing to + // attach across the version mismatch, serves the session in-process. const resp = await waitFor(() => findResponse(server.stdout, 1), 10000); expect(resp.result.serverInfo.name).toBe('codegraph'); await waitFor( - () => server.stderr.some((l) => l.includes('falling back to direct mode')), + () => server.stderr.some((l) => l.includes('serving this session in-process')), 6000, ); } finally { diff --git a/docs/design/dynamic-dispatch-coverage-playbook.md b/docs/design/dynamic-dispatch-coverage-playbook.md index 1124e9f73..ded6d5542 100644 --- a/docs/design/dynamic-dispatch-coverage-playbook.md +++ b/docs/design/dynamic-dispatch-coverage-playbook.md @@ -43,6 +43,7 @@ Static tree-sitter extraction captures explicit calls (`foo()`, `this.bar()`). I | 2 | **Field-backed observer** | `onUpdate(cb)` + `for(cb of cbs)cb()` | callback synthesizer (whole-graph pass) | medium | | 3 | **String-keyed EventEmitter** | `on('e',fn)` / `emit('e')` | callback synthesizer (event-keyed) | medium | | 4 | **Inline callback handler** | `on('e', function h(){})` / `() => {}` | extraction (named) + synthesizer link-through-body (anon) | named: cheap · anon: hard | +| 5 | **Closure-collection dispatch** | Swift `validators.write{$0.append(v)}` … `validators.forEach{$0()}` | callback synthesizer (`closureCollectionEdges`, element-invoke gated) | medium | Key distinction driving the mechanism choice: - **A named ref exists** to resolve (`_iterable_class` is an attribute name) → **resolver**. @@ -78,6 +79,51 @@ Key distinction driving the mechanism choice: extracts named nested functions). - **Result:** `trace(mutateElement, triggerRender)` → 3 hops; express `use → onmount`. +### 3c. Alamofire deferred validation — closure-collection dispatch (Swift) +- **Hole:** `DataRequest.validate(_:)` builds a closure and `validators.write { $0.append(validator) }`; + the base `Request.didCompleteTask` runs them via `validators.forEach { $0() }`. Append and + dispatch live in *different files and classes* (a subclass appends, the base iterates) and the + field is a Swift `Protected<[@Sendable () -> Void]>` — so neither same-file pairing nor the + name-based registrar match (`onX`/`subscribe`/…) reaches it. `trace(didCompleteTask, validate)` + returned no path; the agent grepped `validators` and read three files to reconstruct it. +- **Fix:** `closureCollectionEdges` (callback-synthesizer.ts). A **dispatcher** iterates a collection + *invoking each element* (`coll.forEach { $0() }` / `{ it() }`); a **registrar** appends a closure to + the same-named field (`.append`/`.add`/`.push`/`.insert`, incl. Swift `.write { $0.append }`). The + element-invoke (`$0(` / `it(`) is the precision **gate** — it proves the collection holds closures — + so a repo with no closure-collection dispatch yields **0 edges** regardless of how many `.append` + sites it has. Pairs dispatcher → registrar globally by field name (cross-file/class required), + fan-out-capped. Surfaced two ways: inline in `trace`, and as a "Dynamic-dispatch links among your + symbols" section in `codegraph_explore` (`buildFlowFromNamedSymbols`) so the relationship shows even + when the agent named only `validate`, not the `didCompleteTask` that drains the list. +- **Files:** `src/resolution/callback-synthesizer.ts` (`closureCollectionEdges`), + `src/mcp/tools.ts` (`synthEdgeNote` closure-collection case + the explore synth-links section). +- **Result:** `trace(didCompleteTask, validate)` connects with the closure-collection hop + the + `validators.write { $0.append }` wiring site inlined. 9 precise edges on Alamofire + (`validators`/`streams`/`finishHandlers`/`requestsToRetry`), **0 on every non-Swift control**. + Forced codegraph-only (Read+Grep+Bash blocked): 3/3 runs answer build/send/validate correctly. + +### 3d. Insight — an "adoption floor" can hide a trace-endpoint bug (Alamofire) +Alamofire (110 files) was the README's weakest repo and was written off as the "small-repo floor" +(native grep is cheap, so the agent reads anyway). It wasn't. Reading the **transcripts** — every +`Read`'s `file_path`+offset and the assistant text right before it — surfaced the agent's own words: +*"the trace collided with same-named symbols (44 `request`s, 8 `task`s), let me read by line."* +`codegraph_trace`'s endpoint disambiguation (`scorePair`, shared-dir-prefix only) was resolving an +overloaded name to an **empty delegate/protocol stub** — `request` → `EventMonitor.request(){}` +(a 1-line no-op) over the real `Session.request`, because two unrelated `Source/Features/` stubs +shared a deeper dir prefix than the correct `Source/Core/` pair. Garbage trace → manual reading, +sometimes a spiral (12 reads / 11 greps in one run). **Fix:** a `nodeRelevance` term in `handleTrace` +pair scoring that penalizes empty stubs (≤1 body line) and test-file symbols; among real methods it's +flat, so path-proximity (cosmos `EndBlocker`) is unaffected. Result (n=8): WITH-arm tool calls +12 → 8 median, and the read **variance collapsed** (0–12 → 1–4 — the meltdowns *were* the +trace-collision flounder). General bug: protocol/delegate-stub flooding hits Swift/Java/C#/Go. + +**Methodology lesson:** when the agent reads on a small repo, don't conclude "adoption floor" — diff +*what it read* against what the tool returned *immediately before*. A read of content the tool already +gave = adoption; a read after the tool returned the **wrong thing** (stub endpoints, collided names) = +a fixable bug. The transcript reasoning, not the median, tells you which. The forced codegraph-only +hook (block Read+Grep+Glob+Bash-search) is the variance-free way to confirm sufficiency separately +from adoption. + --- ## 4. The repeatable methodology (run this per language/framework) @@ -188,6 +234,7 @@ Status legend: ✅ done+validated · 🔬 hole identified · ⬜ not started. | Java | MyBatis (XML mappers) | DAO interface method → `` SQL | R (XML extract) + S (Java↔XML synthesizer) | ✅ **XML mapper as first-class language** (#389) — `src/extraction/mybatis-extractor.ts` parses files containing ``; emits one method-shaped node per statement qualified `::` + `` fragments + `` references. Non-mapper XML (pom, log4j) → file node only. `mybatisJavaXmlEdges` synthesizer indexes Java methods by `::` and joins to XML qualified names by suffix-match — ambiguous simple-name collisions dropped (precision over recall). mall-tiny S **6/6 custom-SQL mapper methods bridge** to their XML statements; full enterprise chain `trace(controller.action → mapper.method-xml)` connects across controller / service-iface / impl / mapper / XML. 🔬 cross-mapper `` via unqualified refid; MyBatis Plus dynamic methods (`BaseMapper` CRUD inherited from framework, not in project); annotation-driven mappers (`@Select("SELECT ...")` on Java methods — the SQL lives in the annotation, not XML) | | Kotlin | Spring Boot / Jetpack Compose | request → @RestController → service; @Composable → child | R + X | ✅ **Spring Boot Kotlin** — the Spring resolver was `['java']`-only with a Java-syntax method regex (`public X name()`); extended to `.kt` + Kotlin `fun name(` handler matching (petclinic-kotlin **0→18, 18/18**; class-prefix joins; DI controller→repo resolves — `showOwner ← GET /owners/{ownerId}` → `OwnerRepository.findById`). **Compose composition already static** (@Composable→child are plain function calls — Jetcaster `PodcastInformation→HtmlTextContainer`). Java Spring unchanged (realworld 19/19). 🔬 Ktor `routing { get("/x"){…} }` lambda handlers (anonymous) + Compose recomposition (implicit `mutableStateOf`, no setState gate) + coroutines/Flow | | Swift | Vapor | request → route → controller | R + X | ✅ **was 0 routes on every real app** — the extractor required an `app/router/routes` receiver + a `"path"` literal, but real Vapor routes on grouped builders (`let todos = routes.grouped("todos"); todos.get(use: index)`) with NO path arg. Rewrote: any receiver, optional/non-string path segments, `.grouped`/`.group{}` prefix tracking, `use:` discriminator. vapor-template S **0→3 (3/3**, nested `/todos/:todoID`), SteamPress M **0→27 (27/27)**, SwiftPackageIndex-Server L **0→14 (14/14** handler resolution). 🔬 typed-route enums (SPI `SiteURL.x.pathComponents` — path label only, handler still resolves) + closure handlers `app.get("x"){ }` (anonymous) | +| Swift | Alamofire / closure-collection | request → build → send → **validate** (deferred closures) | S | ✅ **closure-collection dispatch synthesizer** (`closureCollectionEdges`): the Swift deferred-handler pattern `DataRequest.validate` `validators.write{$0.append(v)}` … base `Request.didCompleteTask` `validators.forEach{$0()}` (append + dispatch in different files/classes, field is `Protected<[() -> Void]>`). The element-invoke `$0(`/`it(` is the precision gate → **9 edges on Alamofire** (validators/streams/finishHandlers/requestsToRetry), **0 on every non-closure-collection control**. Surfaced inline in `trace` + as an explore "Dynamic-dispatch links" section (so it shows when the agent named only `validate`, not the `didCompleteTask` that drains the list). Forced codegraph-only: **3/3** build/send/validate correct. + **trace endpoint relevance** (`nodeRelevance`): overloaded `request`/`task` (44/8 defs, mostly empty `EventMonitor` delegate stubs) now resolve to the real `Session.request`, not a 1-line no-op — **WITH-arm tool calls 12→8 median, read variance 0–12→1–4** (the meltdowns were all the trace-collision flounder); control-safe (excalidraw/okhttp/gin traces intact, gin A/B 0 reads). + **god-file multi-phase rendering** (`handleExplore`): a flow whose necessary code spans a god-file (Session.swift build chain ~11K) PLUS other files (validate logic) used to truncate at the fixed `maxOutputChars` and drop whichever phase came last. Six coordinated layers make it render all phases: (1) on-spine god-files render spine-full + off-path methods as signatures (true-spine), (2) every NAMED token's substantive def is seeded into the subgraph (FTS buried `validate` under the build terms → Validation.swift was never gathered), (3) a file that DEFINES a named symbol outranks one that merely references the flow (Validation=50 > incidental Combine=23), (4) the 90%-budget early-break and (5) the total cap both exempt necessary (named/spine) files — incidental files stay capped, (6) the final ceiling is 1.5× so it doesn't slice the necessary content the loop assembled. Alamofire now renders build+validators-exec+validate in ONE explore (~16K); A/B reads med 2→**0.5**, tools 8→**5.5**; excalidraw control held at 0 reads (no bloat). Sequential-flow spine is irreducible (no redundant siblings to collapse) — the fix is to render it, not cap it. | | C# | ASP.NET Core | request → [Http*] action → DI service → EF | X | ✅ **feature-folder detection** (realworld 0→19 — was undetected) + **bare `[HttpGet]` + class `[Route]` prefix** (eShopOnWeb 9→33 / jellyfin L) — co-located so no claimsReference needed. 🔬 EF Core LINQ/DbSet (metaprogramming frontier) | | Ruby | Rails / Sinatra | request → routes.rb → Controller#action → model | R | ✅ **RESTful `resources`/`resource` routing → controller#action** (realworld S 16 / spree M / forem L), pluralization + only/except + claimsReference; explicit routes fixed to precise `controller#action` too. 🔬 ActiveRecord dynamic finders (`Article.find_by_slug`) — metaprogramming frontier | | PHP | Laravel | request → route → controller → Eloquent | R | ✅ **precise `Route::get([Ctrl::class,'m'])` / `'Ctrl@m'` → Ctrl@method** (realworld S / firefly M / bookstack L) — was resolving the bare method name to the WRONG controller (every `index`→ArticleController); Route::resource→controller. 🔬 Eloquent dynamic finders/relationships (metaprogramming frontier) | diff --git a/scripts/agent-eval/parse-bench-readme.mjs b/scripts/agent-eval/parse-bench-readme.mjs index 11affcec0..f7c5ade43 100644 --- a/scripts/agent-eval/parse-bench-readme.mjs +++ b/scripts/agent-eval/parse-bench-readme.mjs @@ -16,7 +16,7 @@ const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alam function parse(file) { if (!existsSync(file)) return null; const L = readFileSync(file, 'utf8').split('\n').filter(Boolean); - let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null; + let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null, raced = false; for (const l of L) { let e; try { e = JSON.parse(l); } catch { continue; } if (e.type === 'assistant') { const u = e.message?.usage; @@ -30,10 +30,21 @@ function parse(file) { else if (/codegraph/.test(n)) cg++; } } + // MCP cold-start race: the headless agent fired before `codegraph serve --mcp` + // finished registering its tools, so early calls returned "No such tool + // available" and the agent floundered into grep/Read. That measures CodeGraph's + // startup latency, NOT its steady-state value — flag the run so the aggregate + // can exclude it (an artifact of headless first-turn timing, not the tool). + if (e.type === 'user') for (const b of (Array.isArray(e.message?.content) ? e.message.content : [])) { + if (b.type === 'tool_result') { + const t = Array.isArray(b.content) ? b.content.map(c => c.text || '').join('') : (b.content || ''); + if (/No such tool available/.test(t)) raced = true; + } + } if (e.type === 'result') r = e; } if (!r || r.subtype !== 'success') return null; - return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0 }; + return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0, raced }; } const median = (arr) => { const v = [...arr].sort((a, b) => a - b); const n = v.length; return n === 0 ? 0 : n % 2 ? v[(n - 1) / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; }; const fmtTime = (s) => s >= 60 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${Math.round(s)}s`; @@ -45,9 +56,14 @@ const savings = { cost: [], tokens: [], time: [], tools: [] }; for (const repo of REPOS) { const dir = join(ROOT, repo); const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)) : []; - const W = [], WO = []; + // Exclude MCP-cold-start-raced WITH runs by default — they measure a startup + // race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw + // distribution). The WITHOUT arm has no MCP, so it's never raced. + const includeRaced = process.env.CG_INCLUDE_RACED === '1'; + const W = [], WO = []; let racedExcluded = 0; for (const rd of runDirs) { - const w = parse(join(dir, rd, 'run-headless-with.jsonl')); if (w) W.push(w); + const w = parse(join(dir, rd, 'run-headless-with.jsonl')); + if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); } const wo = parse(join(dir, rd, 'run-headless-without.jsonl')); if (wo) WO.push(wo); } if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete: w=${W.length} wo=${WO.length})`); continue; } @@ -60,7 +76,8 @@ for (const repo of REPOS) { `${(fmtTime(wT) + '→' + fmtTime(woT)).padEnd(22)}` + `${(Math.round(wTl) + '→' + Math.round(woTl)).padEnd(12)}` + `${(fmtTok(wTok) + '→' + fmtTok(woTok) + ' (' + pct(wTok, woTok) + '%)').padEnd(24)}` + - `$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)` + `$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)` + + (racedExcluded ? ` [${racedExcluded} raced run${racedExcluded === 1 ? '' : 's'} excluded]` : '') ); } const avg = (a) => a.length ? Math.round(a.reduce((s, x) => s + x, 0) / a.length) : 0; diff --git a/src/mcp/engine.ts b/src/mcp/engine.ts index 193f2bbd0..6cd71684e 100644 --- a/src/mcp/engine.ts +++ b/src/mcp/engine.ts @@ -10,10 +10,20 @@ * inotify watch set — that's the entire point of issue #411. */ -import CodeGraph, { findNearestCodeGraphRoot } from '../index'; +import type CodeGraph from '../index'; +import { findNearestCodeGraphRoot } from '../directory'; import { watchDisabledReason } from '../sync'; import { ToolHandler } from './tools'; +// Lazy-load the heavy CodeGraph chain (sqlite + query/graph/context layers) OFF +// the MCP startup path. It's only needed once a tool actually opens a project — +// not to answer initialize/tools-list — so deferring it lets `serve --mcp` (and +// the daemon it spawns) bind + register tools in ~Node-startup time instead of +// ~800ms, closing the "No such tool available" cold-start race that made headless +// agents flounder. require() is sync + cached on the CommonJS build. +const loadCodeGraph = (): typeof import('../index').default => + (require('../index') as typeof import('../index')).default; + export interface MCPEngineOptions { /** * Whether to start the file watcher when initializing. Daemon and direct @@ -118,7 +128,7 @@ export class MCPEngine { try { this.cg.close(); } catch { /* ignore */ } this.cg = null; } - this.cg = CodeGraph.openSync(resolvedRoot); + this.cg = loadCodeGraph().openSync(resolvedRoot); this.projectPath = resolvedRoot; this.toolHandler.setDefaultCodeGraph(this.cg); this.startWatching(); @@ -154,7 +164,7 @@ export class MCPEngine { this.projectPath = resolvedRoot; try { - this.cg = await CodeGraph.open(resolvedRoot); + this.cg = await loadCodeGraph().open(resolvedRoot); this.toolHandler.setDefaultCodeGraph(this.cg); this.startWatching(); this.catchUpSync(); diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 85c3949e2..660687491 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -37,8 +37,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { spawn, StdioOptions } from 'child_process'; -import { findNearestCodeGraphRoot } from '../index'; -import { getCodeGraphDir } from '../directory'; +import { findNearestCodeGraphRoot, getCodeGraphDir } from '../directory'; import { StdioTransport } from './transport'; import { MCPEngine } from './engine'; import { MCPSession } from './session'; @@ -48,7 +47,7 @@ import { isProcessAlive, tryAcquireDaemonLock, } from './daemon'; -import { runProxy } from './proxy'; +import { connectWithHello, runLocalHandshakeProxy } from './proxy'; import { getDaemonSocketPath } from './daemon-paths'; import { HOST_PPID_ENV } from '../extraction/wasm-runtime-flags'; @@ -82,8 +81,14 @@ const TAKEOVER_RETRY_DELAY_MS = 100; * process startup. 60 × 100ms = 6s of headroom for a cold/slow box; on the * common path the socket appears within a few rounds. */ -const DAEMON_CONNECT_MAX_RETRIES = 60; -const DAEMON_CONNECT_RETRY_DELAY_MS = 100; +// Poll finely (25ms) so the proxy attaches the instant the freshly-spawned +// daemon binds, instead of waiting up to a coarse 100ms after — shaves the +// cold-start handshake (the window the headless agent races). Same ~6s total +// give-up budget (240 × 25ms), just finer granularity; socket-connect probes +// are cheap. Paired with deferring the CodeGraph load (engine.ts) off the bind +// path, this narrows the "No such tool available" race window. +const DAEMON_CONNECT_MAX_RETRIES = 240; +const DAEMON_CONNECT_RETRY_DELAY_MS = 25; /** * Resolve the PPID watchdog poll interval from an env override. A value of @@ -258,21 +263,20 @@ export class MCPServer { } try { - const mode = await this.connectOrSpawnDaemon(root); - if (mode === 'fallback') { - return this.startDirect('daemon unavailable; fallback to direct'); - } - // 'proxy': connectOrSpawnDaemon ran the stdio↔socket pipe to completion - // (it only returns once the host disconnected). The process is now - // expected to terminate naturally — the proxy installed its own watchdog. + // Answer the MCP handshake LOCALLY (instant tool registration — no waiting + // ~600ms for the daemon to spawn+bind, which produced the cold-start race) + // and forward tool CALLS to the shared daemon, connected in the background. + // Runs until the host disconnects; the proxy installs its own watchdog and + // falls back to an in-process engine if the daemon never comes up. this.mode = 'proxy'; + await this.runProxyWithLocalHandshake(root); return; } catch (err) { - // Belt-and-braces: if anything throws inside the daemon machinery, - // never wedge the user — fall back to a working direct-mode session. + // Belt-and-braces: a throw during proxy SETUP (before the client was served) + // is still safe to recover from with a direct-mode session. const msg = err instanceof Error ? err.message : String(err); - process.stderr.write(`[CodeGraph MCP] Daemon path failed (${msg}); falling back to direct mode.\n`); - return this.startDirect('daemon path threw'); + process.stderr.write(`[CodeGraph MCP] Proxy path failed (${msg}); falling back to direct mode.\n`); + return this.startDirect('proxy path threw'); } } @@ -376,32 +380,31 @@ export class MCPServer { } /** - * Become a proxy to the shared daemon, spawning the daemon first if none is - * reachable. Returns 'proxy' once the proxied session has run to completion - * (the host disconnected), or 'fallback' if the caller should run in-process. + * Proxy mode (the common case). Serve the MCP handshake LOCALLY for instant + * tool registration, forwarding tool calls to the shared daemon — which is + * connected in the background (probed, then spawned + polled if absent) so the + * handshake never waits ~600ms on it. Runs until the host disconnects; the + * proxy falls back to an in-process engine if the daemon never binds, so this + * never wedges a session. */ - private async connectOrSpawnDaemon(root: string): Promise<'proxy' | 'fallback'> { + private async runProxyWithLocalHandshake(root: string): Promise { const socketPath = getDaemonSocketPath(root); - - // Fast path: a daemon may already be listening. On success runProxy pipes - // stdio until the host disconnects, so a 'proxied' outcome means this - // process has finished its entire job. - let probe = await runProxy(socketPath); - if (probe.outcome === 'proxied') return 'proxy'; - if (probe.reason === 'version mismatch') return 'fallback'; - - // No reachable daemon — spawn one (detached) and wait for it to bind. - spawnDetachedDaemon(root); - - for (let attempt = 0; attempt < DAEMON_CONNECT_MAX_RETRIES; attempt++) { - await sleep(DAEMON_CONNECT_RETRY_DELAY_MS); - probe = await runProxy(socketPath); - if (probe.outcome === 'proxied') return 'proxy'; - if (probe.reason === 'version mismatch') return 'fallback'; - } - - // Daemon never came up in time — run in-process so the user is never blocked. - return 'fallback'; + const getDaemonSocket = async () => { + // Fast path: a daemon may already be listening. + const probe = await connectWithHello(socketPath); + if (probe === 'version-mismatch') return null; // definitive — serve in-process, don't poll for 6s + if (probe) return probe; + // None reachable — spawn one (detached) and poll for its bind. + spawnDetachedDaemon(root); + for (let attempt = 0; attempt < DAEMON_CONNECT_MAX_RETRIES; attempt++) { + await sleep(DAEMON_CONNECT_RETRY_DELAY_MS); + const s = await connectWithHello(socketPath); + if (s === 'version-mismatch') return null; + if (s) return s; + } + return null; // never bound — the proxy serves this session in-process + }; + await runLocalHandshakeProxy({ getDaemonSocket, makeEngine: () => new MCPEngine(), root }); } /** Standard SIGINT/SIGTERM handlers that route to our `stop()` (direct mode). */ diff --git a/src/mcp/proxy.ts b/src/mcp/proxy.ts index 938b135ef..36f72e2d4 100644 --- a/src/mcp/proxy.ts +++ b/src/mcp/proxy.ts @@ -23,6 +23,10 @@ import * as net from 'net'; import { HOST_PPID_ENV } from '../extraction/wasm-runtime-flags'; import { DaemonHello, MAX_HELLO_LINE_BYTES } from './daemon'; import { CodeGraphPackageVersion } from './version'; +import { SERVER_INFO, PROTOCOL_VERSION } from './session'; +import { SERVER_INSTRUCTIONS } from './server-instructions'; +import { getStaticTools } from './tools'; +import type { MCPEngine } from './engine'; /** Default poll cadence for the PPID watchdog (same as the direct server). */ const DEFAULT_PPID_POLL_MS = 5000; @@ -96,6 +100,198 @@ export async function runProxy( process.exit(0); } +/** + * Connect to a daemon at `socketPath` and verify its hello (exact version match). + * Returns the live socket (hello already consumed) or null if unreachable / stale + * / version-mismatched. Unlike {@link runProxy} it does NOT pipe — the caller + * owns the socket. Used by the local-handshake proxy's background connect. + */ +export async function connectWithHello( + socketPath: string, + expectedVersion: string = CodeGraphPackageVersion, +): Promise { + if (process.platform !== 'win32' && !fs.existsSync(socketPath)) return null; + const socket = net.createConnection(socketPath); + socket.setEncoding('utf8'); + const hello = await readHelloLine(socket).catch(() => null); + if (!hello) { + socket.destroy(); + return null; // no daemon yet — caller should keep polling + } + if (hello.codegraph !== expectedVersion) { + // A daemon IS up but it's the wrong version — definitive, not a "not yet". + // Don't poll; the caller serves in-process so we never run stale-vs-new. + process.stderr.write( + `[CodeGraph MCP] Found a daemon on ${socketPath} but version (${hello.codegraph}) ` + + `differs from ours (${expectedVersion}); serving this session in-process.\n` + ); + socket.destroy(); + return 'version-mismatch'; + } + process.stderr.write( + `[CodeGraph MCP] Attached to shared daemon on ${socketPath} (pid ${hello.pid}, v${hello.codegraph}).\n` + ); + return socket; +} + +type JsonRpc = Record; + +/** Dependencies the local-handshake proxy needs, injected by MCPServer (which + * owns the daemon-spawn machinery and the engine factory). */ +export interface LocalHandshakeDeps { + /** Probe → spawn → retry → hello-verify; resolves a connected daemon socket, + * or null when the daemon path is genuinely unavailable (→ in-process fallback). */ + getDaemonSocket(): Promise; + /** Lazily create an in-process engine — used ONLY if the daemon never comes up, + * preserving the "a broken daemon never wedges a session" guarantee. */ + makeEngine(): MCPEngine; + /** Project root for the fallback engine's lazy init. */ + root: string; +} + +/** + * Local-handshake proxy (the cold-start fix). + * + * Answers `initialize` + `tools/list` from STATIC constants the instant the + * client asks — tools register in ~process-startup time instead of waiting + * ~600ms for the daemon to spawn+bind, which is what produced the "No such tool + * available" race that made headless agents flail into grep/Read. Tool CALLS are + * forwarded to the shared daemon (connected in the background); the daemon's + * response to the forwarded `initialize` is suppressed (the client already got + * the local one). If the daemon never comes up (version mismatch / spawn fail), + * a lazily-created in-process engine serves the calls — so the handshake speedup + * never costs the old fall-back-to-direct robustness. + */ +export async function runLocalHandshakeProxy(deps: LocalHandshakeDeps): Promise { + let daemonStatus: 'connecting' | 'ready' | 'failed' = 'connecting'; + let daemonSocket: net.Socket | null = null; + let clientInitId: unknown = undefined; // suppress the daemon's reply to the forwarded initialize + const pending: string[] = []; // client lines buffered until the daemon resolves + let engine: MCPEngine | null = null; + let engineReady: Promise | null = null; + let shuttingDown = false; + + const writeClient = (obj: JsonRpc | string): void => { + try { process.stdout.write((typeof obj === 'string' ? obj : JSON.stringify(obj)) + '\n'); } catch { /* host gone */ } + }; + const shutdown = (): void => { + if (shuttingDown) return; shuttingDown = true; + try { daemonSocket?.destroy(); } catch { /* ignore */ } + try { engine?.stop(); } catch { /* ignore */ } + process.exit(0); + }; + const ensureEngine = (): Promise => { + if (!engine) engine = deps.makeEngine(); + if (!engineReady) engineReady = engine.ensureInitialized(deps.root).catch(() => { /* degraded */ }); + return engineReady; + }; + // Daemon-unavailable fallback: serve a client message in-process. + const handleLocally = async (line: string): Promise => { + let msg: JsonRpc; try { msg = JSON.parse(line) as JsonRpc; } catch { return; } + const id = msg.id; + if (msg.method === 'tools/call' && id !== undefined) { + try { + await ensureEngine(); + const params = (msg.params || {}) as { name: string; arguments?: Record }; + const result = await engine!.getToolHandler().execute(params.name, params.arguments || {}); + writeClient({ jsonrpc: '2.0', id, result }); + } catch (err) { + writeClient({ jsonrpc: '2.0', id, error: { code: -32603, message: err instanceof Error ? err.message : String(err) } }); + } + } else if (msg.method === 'ping' && id !== undefined) { + writeClient({ jsonrpc: '2.0', id, result: {} }); + } + // initialize already answered locally; notifications (initialized) need no reply. + }; + const routeToDaemon = (line: string): void => { + if (daemonStatus === 'ready' && daemonSocket) { + try { daemonSocket.write(line.endsWith('\n') ? line : line + '\n'); } catch { /* close path */ } + } else if (daemonStatus === 'failed') { + void handleLocally(line); + } else { + pending.push(line); + } + }; + + // ---- client (stdin) ---- + let stdinBuf = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk: string) => { + stdinBuf += chunk; + let idx: number; + while ((idx = stdinBuf.indexOf('\n')) !== -1) { + const line = stdinBuf.slice(0, idx).trim(); + stdinBuf = stdinBuf.slice(idx + 1); + if (!line) continue; + let msg: JsonRpc; try { msg = JSON.parse(line) as JsonRpc; } catch { routeToDaemon(line); continue; } + if (msg.method === 'initialize') { + clientInitId = msg.id; + writeClient({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: PROTOCOL_VERSION, capabilities: { tools: {} }, serverInfo: SERVER_INFO, instructions: SERVER_INSTRUCTIONS } }); + routeToDaemon(line); // prime the daemon so it resolves the project (its reply is suppressed below) + } else if (msg.method === 'tools/list') { + writeClient({ jsonrpc: '2.0', id: msg.id, result: { tools: getStaticTools() } }); + } else { + routeToDaemon(line); + } + } + }); + process.stdin.on('end', shutdown); + process.stdin.on('close', shutdown); + startPpidWatchdogNoSocket(shutdown); + + // ---- daemon connection (background) ---- + let socket: net.Socket | null = null; + try { socket = await deps.getDaemonSocket(); } catch { socket = null; } + + if (socket && !shuttingDown) { + daemonSocket = socket; + daemonStatus = 'ready'; + let sockBuf = ''; + socket.setEncoding('utf8'); + socket.on('data', (chunk: string) => { + sockBuf += chunk; + let idx: number; + while ((idx = sockBuf.indexOf('\n')) !== -1) { + const line = sockBuf.slice(0, idx); + sockBuf = sockBuf.slice(idx + 1); + if (!line.trim()) continue; + if (clientInitId !== undefined) { + try { const m = JSON.parse(line) as JsonRpc; if (m.id === clientInitId && ('result' in m || 'error' in m)) continue; } catch { /* relay */ } + } + writeClient(line); + } + }); + socket.on('close', shutdown); + socket.on('error', shutdown); + for (const line of pending) { try { socket.write(line + '\n'); } catch { /* ignore */ } } + pending.length = 0; + } else if (!shuttingDown) { + daemonStatus = 'failed'; + process.stderr.write('[CodeGraph MCP] Shared daemon unavailable; serving this session in-process (degraded).\n'); + const buffered = pending.splice(0); + for (const line of buffered) await handleLocally(line); + } + + await new Promise(() => { /* stdin keeps the loop alive; exit via shutdown() */ }); +} + +/** PPID watchdog for the local-handshake proxy — same #277 logic as + * {@link startPpidWatchdog} but with no socket to close (the caller's shutdown + * handles teardown). */ +function startPpidWatchdogNoSocket(onDeath: () => void): void { + const pollMs = parsePollMs(process.env.CODEGRAPH_PPID_POLL_MS); + if (pollMs <= 0) return; + const originalPpid = process.ppid; + const hostPpid = parseHostPpid(process.env[HOST_PPID_ENV]); + const timer = setInterval(() => { + if (process.ppid !== originalPpid || (hostPpid !== null && !isProcessAliveLocal(hostPpid))) { + process.stderr.write('[CodeGraph MCP] Parent process exited; shutting down.\n'); + onDeath(); + } + }, pollMs); + timer.unref?.(); +} + /** * Read one CRLF/LF-terminated JSON line from the socket, parse it as the * daemon hello, and return it. Bounded to {@link MAX_HELLO_LINE_BYTES} so a diff --git a/src/mcp/session.ts b/src/mcp/session.ts index 157dc17db..beb957bb4 100644 --- a/src/mcp/session.ts +++ b/src/mcp/session.ts @@ -23,13 +23,15 @@ import { CodeGraphPackageVersion } from './version'; * MCP Server Info — kept on the session because some clients log it. The * version tracks the real package version (was a hard-coded '0.1.0'). */ -const SERVER_INFO = { +// Exported so the proxy can answer `initialize` locally with the IDENTICAL +// payload the daemon would send — no drift between the two handshake paths. +export const SERVER_INFO = { name: 'codegraph', version: CodeGraphPackageVersion, }; /** MCP Protocol Version (latest the server claims). */ -const PROTOCOL_VERSION = '2024-11-05'; +export const PROTOCOL_VERSION = '2024-11-05'; /** * How long to wait for the client's `roots/list` response before giving up diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 2e9c6f816..38e343b15 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -4,7 +4,15 @@ * Defines the tools exposed by the CodeGraph MCP server. */ -import CodeGraph, { findNearestCodeGraphRoot } from '../index'; +import type CodeGraph from '../index'; +import { findNearestCodeGraphRoot } from '../directory'; +// Lazy-load the heavy CodeGraph chain off the MCP startup path — see the same +// helper in engine.ts. ToolHandler must load to answer tools/list (static +// schemas), but it must NOT drag in sqlite/query layers before the daemon binds; +// CodeGraph is pulled in only when a tool actually opens a project. require() is +// sync + cached (CommonJS build). +const loadCodeGraph = (): typeof import('../index').default => + (require('../index') as typeof import('../index')).default; import { detectWorktreeIndexMismatch, worktreeMismatchWarning, @@ -622,6 +630,19 @@ export const tools: ToolDefinition[] = [ }, ]; +/** + * Allowlist-filtered tool definitions WITHOUT an engine — the static surface the + * proxy answers `tools/list` with before any project is open. Mirrors + * `ToolHandler.getTools()` in the no-CodeGraph case (the dynamic per-repo budget + * note in a description only adds once `cg` is loaded; the schemas are static). + */ +export function getStaticTools(): ToolDefinition[] { + const raw = process.env.CODEGRAPH_MCP_TOOLS; + if (!raw || !raw.trim()) return tools; + const allow = new Set(raw.split(',').map(s => s.trim().replace(/^codegraph_/, '')).filter(Boolean)); + return allow.size ? tools.filter(t => allow.has(t.name.replace(/^codegraph_/, ''))) : tools; +} + /** * Tool handler that executes tools against a CodeGraph instance * @@ -841,7 +862,7 @@ export class ToolHandler { } // Open and cache under both paths - const cg = CodeGraph.openSync(resolvedRoot); + const cg = loadCodeGraph().openSync(resolvedRoot); this.projectCache.set(resolvedRoot, cg); if (projectPath !== resolvedRoot) { this.projectCache.set(projectPath, cg); @@ -1586,10 +1607,28 @@ export class ToolHandler { - (isLessCanonicalPath(b) ? LESS_CANONICAL_PENALTY : 0); const fromCands = fromMatches.nodes; const toCands = toMatches.nodes; + // Candidate relevance: an overloaded name (Alamofire has 44 `request`s, most + // of them EMPTY EventMonitor protocol-conformance stubs `func request(…){}`) + // floods the pool with no-op decls. Shared-dir-prefix alone then MISLEADS — + // two unrelated `Source/Features/` delegate stubs outscore the real + // `Source/Core/Session.request` × `Source/Core/…task` pair the agent meant, + // so trace resolves to stubs, finds no path, and the agent reads by line. + // Penalize empty stubs and test-file symbols so a substantive entry point + // wins; among real methods this is ~flat, so path-proximity still decides + // (cosmos EndBlocker disambiguation is unaffected — none of its candidates + // are stubs/tests). + const isTestPath = (p: string): boolean => /(^|\/)(tests?|specs?|__tests__|testdata|mocks?|fixtures?)\//i.test(p) || /\.(test|spec)\.[a-z]+$/i.test(p); + const nodeRelevance = (n: Node): number => { + const bodyLines = Math.max(0, (n.endLine ?? n.startLine) - n.startLine); + let s = Math.min(bodyLines, 20); // a substantive body is more likely the meant symbol + if (bodyLines <= 1) s -= 40; // empty/one-line stub (protocol no-op, decl-only) — almost never the trace endpoint + if (isTestPath(n.filePath)) s -= 150; // a Source/ symbol is meant over a Tests/ same-named one + return s; + }; const pairs: Array<{ f: Node; t: Node; score: number }> = []; for (const f of fromCands) { for (const t of toCands) { - pairs.push({ f, t, score: scorePair(f.filePath, t.filePath) }); + pairs.push({ f, t, score: scorePair(f.filePath, t.filePath) + nodeRelevance(f) + nodeRelevance(t) }); } } // Sort by shared prefix desc, then by FTS order (already encoded in the @@ -1843,6 +1882,14 @@ export class ToolHandler { registeredAt, }; } + if (m?.synthesizedBy === 'closure-collection') { + const field = m.field ? `\`${String(m.field)}\`` : 'a collection'; + return { + label: `closure collection — runs handlers appended to ${field} (dynamic dispatch)`, + compact: `dynamic: runs ${field} handlers${at}`, + registeredAt, + }; + } return null; } @@ -2001,20 +2048,62 @@ export class ToolHandler { chain.reverse(); if (!best || chain.length > best.length) best = chain; } - if (!best || best.length < 3) return EMPTY; - const out = ['## Flow (call path among the symbols you queried)', '']; - for (let i = 0; i < best.length; i++) { - const step = best[i]!; - if (step.edge) { const sy = this.synthEdgeNote(step.edge); out.push(` ↓ ${sy ? sy.compact : step.edge.kind}`); } - out.push(`${i + 1}. ${step.node.name} (${step.node.filePath}:${step.node.startLine})`); + const hasMain = !!best && best.length >= 3; + const pathIds = new Set((best ?? []).map((s) => s.node.id)); + + // Supplementary: dynamic-dispatch (synthesized) edges incident to a NAMED + // symbol — the indirect hops an agent would otherwise grep/Read to + // reconstruct ("where do the appended `validators` actually run?"). The + // synth edge IS that answer, so surface it even when the OTHER end wasn't + // named (e.g. the agent names `validate` but not the `didCompleteTask` + // that drains the collection). On-topic by construction: only heuristic + // edges touching a symbol the agent named; skipped when the hop already + // shows in the main chain. + const synthLines: string[] = []; + const synthSeen = new Set(); + for (const n of named.values()) { + if (synthLines.length >= 6) break; + for (const { node: other, edge } of [...cg.getCallers(n.id), ...cg.getCallees(n.id)]) { + if (synthLines.length >= 6) break; + if (edge.provenance !== 'heuristic' || other.id === n.id) continue; + if (pathIds.has(edge.source) && pathIds.has(edge.target)) continue; // already in the main chain + const src = edge.source === n.id ? n : other; + const tgt = edge.source === n.id ? other : n; + const key = `${src.name}>${tgt.name}`; + if (synthSeen.has(key)) continue; + synthSeen.add(key); + const note = this.synthEdgeNote(edge); + synthLines.push(`- ${src.name} → ${tgt.name} [${note ? note.compact : edge.kind}]`); + } + } + + if (!hasMain && synthLines.length === 0) return EMPTY; + const out: string[] = []; + if (hasMain) { + out.push('## Flow (call path among the symbols you queried)', ''); + for (let i = 0; i < best!.length; i++) { + const step = best![i]!; + if (step.edge) { const sy = this.synthEdgeNote(step.edge); out.push(` ↓ ${sy ? sy.compact : step.edge.kind}`); } + out.push(`${i + 1}. ${step.node.name} (${step.node.filePath}:${step.node.startLine})`); + } + out.push(''); } - out.push('', '> Full source for these symbols is below; codegraph_trace(from,to) for the exact path between two endpoints.', ''); + if (synthLines.length) { + out.push( + '## Dynamic-dispatch links among your symbols', + '(synthesized — the indirect hops grep/Read would reconstruct; the `@file:line` is the wiring site)', + '', + ...synthLines, + '' + ); + } + out.push('> Full source for these symbols is below; codegraph_trace(from,to) for the exact path between two endpoints.', ''); // namedNodeIds = every callable the agent explicitly named (a superset of // the spine). A file holding one is something the agent asked to SEE, so it // must keep full source even if it's an off-spine polymorphic sibling — the // agent named `getResponseWithInterceptorChain` / `SQLCompiler.execute_sql` // as the mechanism, not as an interchangeable leaf. See the skeleton gate. - return { text: out.join('\n'), pathNodeIds: new Set(best.map((s) => s.node.id)), namedNodeIds: new Set(named.keys()), uniqueNamedNodeIds }; + return { text: out.join('\n'), pathNodeIds: pathIds, namedNodeIds: new Set(named.keys()), uniqueNamedNodeIds }; } catch { return EMPTY; } @@ -2096,9 +2185,42 @@ export class ToolHandler { } } + // Named-symbol seeding: findRelevantContext is an FTS/text rank, so a query + // that's a BAG of symbol names skewed toward one phase (Alamofire: 5 build + // terms, each a high-frequency name, vs 3 validate terms) lets the + // lower-frequency names fall below the search cut — their definitions, and + // whole files (Validation.swift), never get gathered, so they can never + // render and the agent Reads them. Resolve EACH named token to its + // substantive definition (skip empty stubs + test files, same relevance the + // trace endpoint picker uses) and inject it as an entry, so every symbol the + // agent explicitly named is in the subgraph and its file is scored. + const namedSeedIds = new Set(); + { + const FILE_EXT = /\.(?:java|kt|kts|ts|tsx|js|jsx|mjs|cjs|cs|py|go|rb|php|swift|rs|cpp|cc|cxx|c|h|hpp|scala|lua|dart|vue|svelte)$/i; + const CALLABLE = new Set(['method', 'function', 'component', 'constructor']); + const isTestPath = (p: string) => /(^|\/)(tests?|specs?|__tests__|testdata|mocks?|fixtures?)\//i.test(p) || /\.(test|spec)\.[a-z]+$/i.test(p); + const bodyLines = (n: Node) => Math.max(0, (n.endLine ?? n.startLine) - n.startLine); + const tokens = [...new Set( + query.split(/[\s,()[\]]+/) + .map((t) => t.replace(FILE_EXT, '').trim()) + .filter((t) => t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) + )].slice(0, 16); + for (const t of tokens) { + const cands = this.findAllSymbols(cg, t).nodes + .filter((n) => CALLABLE.has(n.kind) && !isTestPath(n.filePath)) + .sort((a, b) => (bodyLines(b) > 1 ? 1 : 0) - (bodyLines(a) > 1 ? 1 : 0) || bodyLines(b) - bodyLines(a)); + // A specific name (<=3 defs) injects all its defs; an overloaded name + // (`request` = 44, mostly stubs) injects only the single most substantive + // one, so the build-overload flood doesn't crowd the subgraph. + for (const n of cands.slice(0, cands.length <= 3 ? cands.length : 1)) { + if (!subgraph.nodes.has(n.id)) { subgraph.nodes.set(n.id, n); namedSeedIds.add(n.id); } + } + } + } + // Step 2: Group nodes by file, score by relevance const fileGroups = new Map(); - const entryNodeIds = new Set(subgraph.roots); + const entryNodeIds = new Set([...subgraph.roots, ...namedSeedIds]); // Build a set of nodes directly connected to entry points (depth 1) const connectedToEntry = new Set(); @@ -2113,8 +2235,15 @@ export class ToolHandler { const group = fileGroups.get(node.filePath) || { nodes: [], score: 0 }; group.nodes.push(node); - // Score: entry point nodes worth 10, directly connected worth 3, others worth 1 - if (entryNodeIds.has(node.id)) { + // Score: a NAMED-SEED node (a symbol the agent named that FTS missed, now + // injected) is worth far more than a mere reference — its file is where the + // answer lives. Without this, an incidental file that name-drops the flow + // (Combine.swift references request/task → score 23 from connected nodes) + // outranks the file that DEFINES a named symbol (Validation.swift's + // `validate` → 10) and steals its render slot. Definition ≫ reference. + if (namedSeedIds.has(node.id)) { + group.score += 50; + } else if (entryNodeIds.has(node.id)) { group.score += 10; } else if (connectedToEntry.has(node.id)) { group.score += 3; @@ -2315,7 +2444,15 @@ export class ToolHandler { for (const [filePath, group] of sortedFiles) { if (filesIncluded >= maxFiles) break; - if (totalChars > budget.maxOutputChars * 0.9) break; + // A file DEFINES a named/spine symbol (the answer) vs merely references the + // flow. Past 90% budget, stop pulling INCIDENTAL files — but keep scanning + // for necessary ones, which render even past the cap (bounded by maxFiles). + // Without this `continue` (was an unconditional `break`), the loop stopped + // after the build + validators-exec files and never reached the ranked-in + // validate-logic file (Alamofire's Validation.swift). + const fileNecessary = group.nodes.some(n => + entryNodeIds.has(n.id) || flow.pathNodeIds.has(n.id) || flow.uniqueNamedNodeIds.has(n.id)); + if (!fileNecessary && totalChars > budget.maxOutputChars * 0.9) continue; const absPath = validatePathWithinRoot(projectRoot, filePath); if (!absPath || !existsSync(absPath)) continue; @@ -2351,11 +2488,25 @@ export class ToolHandler { const spareNamed = group.nodes.some(n => flow.uniqueNamedNodeIds.has(n.id)); const fileDefinesSuper = definesPolymorphicSupertype(group.nodes); const spared = spareNamed && !fileDefinesSuper; + const CALLABLE_BODY = new Set(['method', 'function', 'constructor', 'component']); + const hasSpineNode = group.nodes.some(n => flow.pathNodeIds.has(n.id)); + // On-spine god-file: the flow path runs THROUGH this file, but it also holds + // many OTHER named methods, and rendering all of them in full blows the + // per-file budget and starves the other flow files (Alamofire: the agent + // names ~7 Session.swift methods — the build spine PLUS off-path + // task/didCompleteTask — far past the whole response budget). Engage the + // per-symbol view to keep the SPINE full and collapse the off-path named + // methods to signatures. Only when there IS off-path content to shed — + // otherwise the spine is irreducible (a sequential flow has no redundancy), + // so leave it to the normal full render. + const namedBodyChars = group.nodes + .filter(n => CALLABLE_BODY.has(n.kind) && (flow.pathNodeIds.has(n.id) || flow.uniqueNamedNodeIds.has(n.id))) + .reduce((s, n) => s + fileLines.slice(n.startLine - 1, Math.min(n.endLine, n.startLine + 220)).join('\n').length, 0); + const onSpineGodFile = hasSpineNode + && namedBodyChars > budget.maxCharsPerFile + && group.nodes.some(n => CALLABLE_BODY.has(n.kind) && flow.uniqueNamedNodeIds.has(n.id) && !flow.pathNodeIds.has(n.id)); if (adaptiveExploreEnabled() && flow.pathNodeIds.size > 0 - && !group.nodes.some(n => flow.pathNodeIds.has(n.id)) - && isPolymorphicSibling(group.nodes) - && !spared) { - const CALLABLE_BODY = new Set(['method', 'function', 'constructor', 'component']); + && (onSpineGodFile || (!hasSpineNode && isPolymorphicSibling(group.nodes) && !spared))) { const syms = group.nodes .filter(n => n.kind !== 'import' && n.kind !== 'export' && n.startLine > 0) .sort((a, b) => a.startLine - b.startLine); @@ -2375,7 +2526,9 @@ export class ToolHandler { let bodyChars = 0; for (const n of syms.filter(n => prio(n) < 99 && n.endLine >= n.startLine).sort((a, b) => prio(a) - prio(b))) { const sz = fileLines.slice(n.startLine - 1, Math.min(n.endLine, n.startLine + 220)).join('\n').length; - if (bodyChars + sz > bodyCap && bodyIds.size > 0) continue; + // Spine methods (prio 0) ALWAYS get a full body — the cap governs the + // off-path extras (unique-named, family base), never the flow path itself. + if (prio(n) > 0 && bodyChars + sz > bodyCap && bodyIds.size > 0) continue; bodyIds.add(n.id); bodyChars += sz; } @@ -2410,9 +2563,15 @@ export class ToolHandler { if (skel.length > 0) { const names = [...new Set(group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export').map(n => n.name))] .slice(0, budget.maxSymbolsInFileHeader).join(', '); + // Steer the agent to codegraph_explore for an elided body — NEVER to + // Read. The old "Read for more" / "Read for a full body" tags invited + // a Read of the very file just skeletonized; on a central, wanted file + // (Session.swift, DataRequest.swift) that fired an over-investigation + // spiral (the agent Read the skeletonized file, then kept digging). + // CLAUDE.md: explore output must never tell the agent to Read. const tag = bodyIds.size > 0 - ? 'focused (the methods you named in full, the rest as signatures; Read for more)' - : 'skeleton (signatures only; Read for a full body)'; + ? 'focused (the methods you named in full, the rest as signatures — codegraph_explore a signature by name for its body; do NOT Read)' + : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; lines.push(`#### ${filePath} — ${names} · ${tag}`, '', '```' + lang, skel.join('\n'), '```', ''); totalChars += skel.join('\n').length + 120; filesIncluded++; @@ -2658,22 +2817,21 @@ export class ToolHandler { : headerSymbols.join(', '); const fileHeader = `#### ${filePath} — ${headerSuffix}`; - // Respect the total output cap on a file-by-file basis. - if (totalChars + fileSection.length + 200 > budget.maxOutputChars) { + // The total cap bounds INCIDENTAL files only. A file that DEFINES a symbol + // the agent named (or that's on the flow spine) renders even when the + // nominal total is used up — it's the answer, and the set is bounded by + // maxFiles AND by true-spine/named-seeding having already trimmed each file + // to its necessary content. A file that merely REFERENCES the flow + // (Combine.swift name-drops request/task) is incidental → still capped, so + // freed budget never leaks into noise. This is the last god-file layer: + // build (Session, true-spined) + validators-exec (Request) + validate + // (DataRequest/Validation) all render, instead of the cap dropping whichever + // phase the file order happened to put last. + if (!fileNecessary && totalChars + fileSection.length + 200 > budget.maxOutputChars) { const remaining = budget.maxOutputChars - totalChars - 200; - if (remaining < 500) break; - const trimmed = fileSection.slice(0, remaining) + '\n... (trimmed) ...'; - - lines.push(fileHeader); - lines.push(''); - lines.push('```' + lang); - lines.push(trimmed); - lines.push('```'); - lines.push(''); - totalChars += trimmed.length + 200; - filesIncluded++; + if (remaining < 500) continue; // incidental file, no room — skip it, keep scanning for necessary ones + fileSection = fileSection.slice(0, remaining) + '\n... (trimmed) ...'; anyFileTrimmed = true; - break; } lines.push(fileHeader); @@ -2740,11 +2898,20 @@ export class ToolHandler { // maxOutputChars (observed 30k against a 28k tier cap). A fat explore // payload persists in the agent's context and is re-read as cache-input // on every subsequent turn, so the overrun is paid many times over. + // Final ceiling. The render loop is now the authority on WHAT to emit — it + // renders necessary files (named/spine) even past maxOutputChars and caps + // only incidental ones, all bounded by maxFiles + per-file true-spine — so + // this is a SAFETY ceiling above that necessary content, not a hard cut + // through it. Cutting at a flat maxOutputChars here undid the whole point: + // Alamofire's loop assembles build+validators-exec+validate (~15K) and a 13K + // slice dropped the validate phase the agent then Read. Allow necessary + // overflow up to 1.5× (still bounds a pathological monolith). const output = flow.text + lines.join('\n'); - if (output.length > budget.maxOutputChars) { - const cut = output.slice(0, budget.maxOutputChars); + const hardCeiling = Math.round(budget.maxOutputChars * 1.5); + if (output.length > hardCeiling) { + const cut = output.slice(0, hardCeiling); const lastNewline = cut.lastIndexOf('\n'); - const safe = lastNewline > budget.maxOutputChars * 0.8 ? cut.slice(0, lastNewline) : cut; + const safe = lastNewline > hardCeiling * 0.8 ? cut.slice(0, lastNewline) : cut; return this.textResult(safe + '\n\n... (output truncated to budget; the source above is complete and verbatim — treat it as already Read. For any area not covered, run another codegraph_explore with the specific names — do NOT Read these files.)'); } return this.textResult(output); diff --git a/src/resolution/callback-synthesizer.ts b/src/resolution/callback-synthesizer.ts index 43e7bf0ba..53b05ba53 100644 --- a/src/resolution/callback-synthesizer.ts +++ b/src/resolution/callback-synthesizer.ts @@ -47,6 +47,19 @@ const VUE_HANDLER_RE = /(?:@|v-on:)([a-zA-Z][\w-]*)(?:\.[\w]+)*\s*=\s*"([^"]+)"/ // Captures the destructure body + the called composable; only `use*` calls qualify. const VUE_DESTRUCTURE_RE = /(?:const|let|var)\s*\{([^}]+)\}\s*=\s*(\w+)\s*\(/g; +// Closure-collection dynamic dispatch (language-agnostic, Swift-first). A method +// appends a closure to a collection property; another method iterates that +// property *invoking each element* (`coll.forEach { $0() }` / `{ it() }`). The +// element-invoke (`$0(` / `it(`) PROVES the collection holds closures, so pairing +// a dispatcher to same-named registrars (`.append`/`.add`/`.push`/`.insert`, +// incl. Swift `prop.write { $0.append }`) is high-precision. Cross-file/class by +// design: Alamofire appends in `DataRequest.validate` but iterates in the base +// `Request.didCompleteTask` — neither same-file nor same-class pairing reaches it. +const CC_DISPATCH_RE = /(\w+)\.forEach\s*\{\s*(?:\$0|it)\s*\(/g; +const CC_APPEND_WRITE_RE = /(\w+)\.write\s*\{\s*\$0(?:\.(\w+))?\.(?:append|add|push|insert)\s*\(/g; +const CC_APPEND_DIRECT_RE = /(\w+)\.(?:append|add|push|insert)\s*\(/g; +const CC_FANOUT_CAP = 8; // skip a field name with more dispatchers/registrars than this (too generic to pair confidently) + function kebabToPascal(s: string): string { return s.split('-').map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(''); } @@ -143,6 +156,81 @@ function fieldChannelEdges(queries: QueryBuilder, ctx: ResolutionContext): Edge[ return edges; } +/** + * Closure-collection dispatch: dispatcher iterates a closure-collection property + * invoking each element; registrar appends a closure to the same-named property. + * Emits dispatcher → registrar so a flow reaches the registration site (where the + * appended closure's body — and its callers — live). High-precision: the + * dispatcher's element-invoke is the gate (a `.forEach` that does NOT invoke its + * element is ignored), so a repo with no closure-collection dispatch yields zero + * edges regardless of how many `.append`/`.push` sites it has. + * + * Pairs globally by field name (cross-file/class is required — see Alamofire's + * base-class `Request.didCompleteTask` iterating `validators` appended by the + * subclass `DataRequest.validate`), bounded by a fan-out cap so a generic field + * name shared across unrelated classes can't fan out into noise. + */ +function closureCollectionEdges(queries: QueryBuilder, ctx: ResolutionContext): Edge[] { + const candidates = [...queries.getNodesByKind('method'), ...queries.getNodesByKind('function')]; + const dispatchers = new Map>(); // field → dispatcher methods + forEach line + const registrars = new Map>(); // field → registrar methods + append line + + const addReg = (field: string | undefined, node: Node, absLine: number) => { + if (!field || /^\d+$/.test(field)) return; // `$0.append` mis-captures the `0`; the write-RE owns that field + const arr = registrars.get(field) ?? []; + if (!arr.some((r) => r.node.id === node.id)) arr.push({ node, line: absLine }); + registrars.set(field, arr); + }; + + for (const m of candidates) { + const content = ctx.readFile(m.filePath); + const src = content && sliceLines(content, m.startLine, m.endLine); + if (!src) continue; + const hasForEach = src.includes('.forEach'); + const hasAppend = src.includes('.append(') || src.includes('.add(') || src.includes('.push(') || src.includes('.insert('); + if (!hasForEach && !hasAppend) continue; + const lineAt = (idx: number) => (m.startLine ?? 1) + src.slice(0, idx).split('\n').length - 1; + + if (hasForEach) { + CC_DISPATCH_RE.lastIndex = 0; + let d: RegExpExecArray | null; + while ((d = CC_DISPATCH_RE.exec(src))) { + const arr = dispatchers.get(d[1]!) ?? []; + if (!arr.some((n) => n.node.id === m.id)) arr.push({ node: m, line: lineAt(d.index) }); + dispatchers.set(d[1]!, arr); + } + } + if (hasAppend) { + CC_APPEND_WRITE_RE.lastIndex = 0; + let w: RegExpExecArray | null; + while ((w = CC_APPEND_WRITE_RE.exec(src))) addReg(w[2] || w[1], m, lineAt(w.index)); // nested `$0.streams` else the `.write` receiver + CC_APPEND_DIRECT_RE.lastIndex = 0; + let a: RegExpExecArray | null; + while ((a = CC_APPEND_DIRECT_RE.exec(src))) addReg(a[1], m, lineAt(a.index)); + } + } + + const edges: Edge[] = []; + const seen = new Set(); + for (const [field, disps] of dispatchers) { + const regs = registrars.get(field); + if (!regs || regs.length === 0) continue; + if (disps.length > CC_FANOUT_CAP || regs.length > CC_FANOUT_CAP) continue; // generic field — can't pair confidently + for (const disp of disps) for (const reg of regs) { + if (disp.node.id === reg.node.id) continue; + const key = `${disp.node.id}>${reg.node.id}`; + if (seen.has(key)) continue; + seen.add(key); + edges.push({ + source: disp.node.id, target: reg.node.id, kind: 'calls', line: disp.line, + provenance: 'heuristic', + metadata: { synthesizedBy: 'closure-collection', field, registeredAt: `${reg.node.filePath}:${reg.line}` }, + }); + } + } + return edges; +} + /** Phase 2: string-keyed EventEmitter channels (on('e', fn) ↔ emit('e')). */ function eventEmitterEdges(ctx: ResolutionContext): Edge[] { const emitsByEvent = new Map>(); // event → dispatcher node ids @@ -1093,6 +1181,7 @@ function ginMiddlewareChainEdges(queries: QueryBuilder, ctx: ResolutionContext): */ export function synthesizeCallbackEdges(queries: QueryBuilder, ctx: ResolutionContext): number { const fieldEdges = fieldChannelEdges(queries, ctx); + const closureCollEdges = closureCollectionEdges(queries, ctx); const emitterEdges = eventEmitterEdges(ctx); const renderEdges = reactRenderEdges(queries, ctx); const jsxEdges = reactJsxChildEdges(ctx); @@ -1110,6 +1199,7 @@ export function synthesizeCallbackEdges(queries: QueryBuilder, ctx: ResolutionCo const seen = new Set(); for (const e of [ ...fieldEdges, + ...closureCollEdges, ...emitterEdges, ...renderEdges, ...jsxEdges,