diff --git a/.changeset/index-lock-error-log.md b/.changeset/index-lock-error-log.md new file mode 100644 index 0000000..d3b99e5 --- /dev/null +++ b/.changeset/index-lock-error-log.md @@ -0,0 +1,5 @@ +--- +"@stainless-code/codemap": minor +--- + +Add cross-process index lock with `codemap unlock`, and append parse failures to `/errors.log`. diff --git a/docs/agents.md b/docs/agents.md index 9528eba..6fbe764 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -60,6 +60,8 @@ All integrations reuse the **same** bundled content under **`.agents/`**. Symlin When the file watcher is off (WSL `/mnt/*` mounts, `CODEMAP_WATCH=0`, etc.), **`codemap agents init --git-hooks`** installs marker-delimited blocks in **`post-commit`**, **`post-merge`**, and **`post-checkout`** that run `( codemap >/dev/null 2>&1 & )` — non-blocking background incremental index. **`--no-git-hooks`** removes only codemap-marked blocks. Interactive init offers hooks automatically when [`watch-policy.ts`](../src/application/watch-policy.ts) would disable the watcher for the project root. +Concurrent indexers (CLI, MCP `--watch`, git hooks) coordinate via **`/index.lock`**. If indexing fails with “Index already running” after a crash, run **`codemap unlock`**. Per-file parse failures append to **`/errors.log`**. + ## Pointer files Root / Copilot **pointer** files (**`CLAUDE.md`**, **`AGENTS.md`**, **`GEMINI.md`**, **`.github/copilot-instructions.md`**) use a **managed section** between **``** and **``** (HTML comments — usually hidden in rendered Markdown): diff --git a/docs/plans/agent-surface-delivery.md b/docs/plans/agent-surface-delivery.md index c237a19..f621f28 100644 --- a/docs/plans/agent-surface-delivery.md +++ b/docs/plans/agent-surface-delivery.md @@ -10,11 +10,11 @@ ## Quick resume -| Next action | Detail | -| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Review / merge P0** | PR [#126](https://github.com/stainless-code/codemap/pull/126) (MCP instructions + allowlist), PR [#127](https://github.com/stainless-code/codemap/pull/127) (WSL watch + git hooks) | -| **Start next** | **PR 3** — [`index-lock-and-error-log`](./index-lock-and-error-log.md) on branch from fresh `main` | -| **Do not start yet** | PR 6 (MCP trace tools) until PR 1 + PR 4 land; PR 9 (eval harness) until PR 1 + PR 8 | +| Next action | Detail | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| **Review / merge** | PR 3 — index lock + error log (branch `feat/index-lock`) when open | +| **Start next** | **PR 4** — trace recipes (`call-path`, `symbol-neighborhood`) or **PR 5** — `affected-tests-recipe` (parallel with 4 after 3 merges) | +| **Do not start yet** | PR 6 (MCP trace tools) until PR 4 land; PR 9 (eval harness) until PR 8 | Update the table below when a PR merges or a new branch opens. @@ -26,24 +26,24 @@ Merge each PR to `main` directly. No long-lived integration branch (`feat/agent- ### Wave 1 — P0 (~1 week wall clock, 2 PRs) -| PR | Plans bundled | Status | Branch / link | Notes | -| ----- | ----------------------------------------------------------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| **1** | [`mcp-server-instructions`](./mcp-server-instructions.md) + [`mcp-tool-allowlist`](./mcp-tool-allowlist.md) | open | [`feat/mcp-instructions-allowlist`](https://github.com/stainless-code/codemap/pull/126) | Same hot files (`mcp-server.ts`, `agent-content`); ~3–4 days | -| **2** | [`wsl-watch-policy`](./wsl-watch-policy.md) → [`git-hook-auto-sync`](./git-hook-auto-sync.md) | open | [`feat/wsl-watch-git-hooks`](https://github.com/stainless-code/codemap/pull/127) | `watch-policy.ts` first; hooks reference it in diagnostics. Lock deferred to PR 3 — note concurrent hook + MCP in PR 2 body | +| PR | Plans bundled | Status | Branch / link | Notes | +| ----- | ----------------------------------------------------------------------------------------------------------- | ------ | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| **1** | [`mcp-server-instructions`](./mcp-server-instructions.md) + [`mcp-tool-allowlist`](./mcp-tool-allowlist.md) | merged | [#126](https://github.com/stainless-code/codemap/pull/126) | Same hot files (`mcp-server.ts`, `agent-content`); ~3–4 days | +| **2** | [`wsl-watch-policy`](./wsl-watch-policy.md) → [`git-hook-auto-sync`](./git-hook-auto-sync.md) | merged | [#127](https://github.com/stainless-code/codemap/pull/127) | `watch-policy.ts` first; hooks reference it in diagnostics. Lock deferred to PR 3 — note concurrent hook + MCP in PR 2 body | ### Wave 2 — P1 (~2–3 weeks, parallel tracks) Max **3 parallel tracks** at once. -| PR | Plans | Status | Blocked by | Parallel with | -| ----- | ----------------------------------------------------------------------------------------------------------------------------- | ------- | ----------------------------------- | --------------------------------- | -| **3** | [`index-lock-and-error-log`](./index-lock-and-error-log.md) → [`parse-worker-hardening`](./parse-worker-hardening.md) (stack) | planned | PR 2 merged (doc note on hook+lock) | 4, 5 | -| **4** | Recipe half of [`mcp-trace-explore-tools`](./mcp-trace-explore-tools.md) (`call-path`, `symbol-neighborhood` SQL + tests) | planned | — | 3, 5 | -| **5** | [`affected-tests-recipe`](./affected-tests-recipe.md) | planned | — | 3, 4 | -| **6** | MCP half of trace (`trace` / `explore` / `node` tools) + update instructions | planned | PR 1, PR 4 | — | -| **7** | [`field-qualified-search`](./field-qualified-search.md) | planned | PR 1 | 4, 5 if `mcp-server.ts` untouched | -| **8** | [`agents-init-mcp-wiring`](./agents-init-mcp-wiring.md) | planned | PR 1 | 3–5 | -| **9** | [`agent-eval-harness`](./agent-eval-harness.md) | planned | PR 1, PR 8, allowlist | **last P1** | +| PR | Plans | Status | Blocked by | Parallel with | +| ----- | ----------------------------------------------------------------------------------------------------------------------------- | ------- | --------------------- | --------------------------------- | +| **3** | [`index-lock-and-error-log`](./index-lock-and-error-log.md) → [`parse-worker-hardening`](./parse-worker-hardening.md) (stack) | open | PR 2 merged | 4, 5 | +| **4** | Recipe half of [`mcp-trace-explore-tools`](./mcp-trace-explore-tools.md) (`call-path`, `symbol-neighborhood` SQL + tests) | planned | — | 3, 5 | +| **5** | [`affected-tests-recipe`](./affected-tests-recipe.md) | planned | — | 3, 4 | +| **6** | MCP half of trace (`trace` / `explore` / `node` tools) + update instructions | planned | PR 1, PR 4 | — | +| **7** | [`field-qualified-search`](./field-qualified-search.md) | planned | PR 1 | 4, 5 if `mcp-server.ts` untouched | +| **8** | [`agents-init-mcp-wiring`](./agents-init-mcp-wiring.md) | planned | PR 1 | 3–5 | +| **9** | [`agent-eval-harness`](./agent-eval-harness.md) | planned | PR 1, PR 8, allowlist | **last P1** | **Parallelization constraints** diff --git a/src/application/error-log.ts b/src/application/error-log.ts new file mode 100644 index 0000000..3bc5e44 --- /dev/null +++ b/src/application/error-log.ts @@ -0,0 +1,21 @@ +import { appendFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; + +/** Append-only per-file index failure log inside `/`. */ +export const ERROR_LOG_NAME = "errors.log"; + +export function errorLogPath(stateDir: string): string { + return join(stateDir, ERROR_LOG_NAME); +} + +/** One TSV line: ISO timestamp, file path, reason (tabs flattened). */ +export function appendIndexError( + stateDir: string, + filePath: string, + reason: string, +): void { + mkdirSync(stateDir, { recursive: true }); + const safeReason = reason.replace(/\t/g, " ").replace(/\n/g, " "); + const line = `${new Date().toISOString()}\t${filePath}\t${safeReason}\n`; + appendFileSync(errorLogPath(stateDir), line, "utf-8"); +} diff --git a/src/application/index-engine.ts b/src/application/index-engine.ts index 187545f..467b411 100644 --- a/src/application/index-engine.ts +++ b/src/application/index-engine.ts @@ -57,6 +57,7 @@ import { getFts5Enabled, getIncludePatterns, getProjectRoot, + getStateDir, isPathExcluded, } from "../runtime"; import { parseFilesParallel } from "../worker-pool"; @@ -67,6 +68,7 @@ import { resolveBindings, } from "./bindings-engine"; import { persistModuleCycles } from "./cycles-engine"; +import { appendIndexError } from "./error-log"; import { persistFileBarrelFlags } from "./file-graph-flags"; import { persistJsxElementsAndAttributes } from "./jsx-persist"; import type { QueryBindValue } from "./query-engine"; @@ -273,6 +275,15 @@ export function getCurrentCommit(): string { return result.stdout.toString().trim(); } +function reportParseError(relPath: string, reason: string): void { + console.error(` Parse error in ${relPath}: ${reason}`); + try { + appendIndexError(getStateDir(), relPath, reason); + } catch { + // logging must not fail the index run + } +} + function insertParsedResults( db: CodemapDatabase, results: ParsedFile[], @@ -283,7 +294,14 @@ function insertParsedResults( const transaction = db.transaction(() => { for (const parsed of results) { - if (parsed.error) continue; + if (parsed.error) { + reportParseError(parsed.relPath, "file read failed"); + continue; + } + + if (parsed.parseError) { + reportParseError(parsed.relPath, parsed.parseError); + } if (parsed.hasSideEffects) { parsed.fileRow.has_side_effects = parsed.hasSideEffects; @@ -364,8 +382,9 @@ function insertParsedResults( if (parsed.suppressions?.length) insertSuppressions(db, parsed.suppressions); } catch (err) { - console.error( - ` Parse error in ${parsed.relPath}: ${err instanceof Error ? err.message : err}`, + reportParseError( + parsed.relPath, + err instanceof Error ? err.message : String(err), ); } @@ -600,8 +619,9 @@ export async function indexFiles( const suppressions = extractSuppressions(source, relPath); if (suppressions.length) insertSuppressions(db, suppressions); } catch (err) { - console.error( - ` Parse error in ${relPath}: ${err instanceof Error ? err.message : err}`, + reportParseError( + relPath, + err instanceof Error ? err.message : String(err), ); } diff --git a/src/application/index-lock.test.ts b/src/application/index-lock.test.ts new file mode 100644 index 0000000..b5c2a94 --- /dev/null +++ b/src/application/index-lock.test.ts @@ -0,0 +1,111 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +import { appendIndexError, errorLogPath } from "./error-log"; +import { + IndexLockHeldError, + acquireIndexLock, + indexLockPath, + isStaleLock, + readIndexLock, + removeIndexLock, +} from "./index-lock"; + +function scratchDir(prefix: string): string { + const base = join(process.cwd(), "fixtures", "tmp"); + mkdirSync(base, { recursive: true }); + return mkdtempSync(join(base, prefix)); +} + +describe("index-lock", () => { + let dir: string; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it("acquireIndexLock creates JSON lock and release removes it", () => { + dir = scratchDir("index-lock-"); + const release = acquireIndexLock(dir); + const lockPath = indexLockPath(dir); + expect(existsSync(lockPath)).toBe(true); + const payload = readIndexLock(lockPath); + expect(payload?.pid).toBe(process.pid); + release(); + expect(existsSync(lockPath)).toBe(false); + }); + + it("second acquire fails fast while lock is held", () => { + dir = scratchDir("index-lock-held-"); + const release = acquireIndexLock(dir); + try { + expect(() => acquireIndexLock(dir)).toThrow(IndexLockHeldError); + } finally { + release(); + } + }); + + it("isStaleLock treats dead PID as stale", () => { + expect( + isStaleLock({ pid: 999_999_999, started_at: new Date().toISOString() }), + ).toBe(true); + }); + + it("removeIndexLock refuses live lock without force", () => { + dir = scratchDir("index-lock-live-"); + const release = acquireIndexLock(dir); + try { + expect(() => removeIndexLock(dir)).toThrow(IndexLockHeldError); + } finally { + release(); + } + }); + + it("removeIndexLock with force clears a live lock", () => { + dir = scratchDir("index-lock-force-"); + const release = acquireIndexLock(dir); + expect(removeIndexLock(dir, { force: true })).toBe(true); + expect(existsSync(indexLockPath(dir))).toBe(false); + release(); + }); + + it("auto-steals stale lock on acquire", () => { + dir = scratchDir("index-lock-stale-"); + const lockPath = indexLockPath(dir); + mkdirSync(dir, { recursive: true }); + writeFileSync( + lockPath, + `${JSON.stringify({ + pid: 999_999_999, + started_at: new Date().toISOString(), + })}\n`, + "utf-8", + ); + const release = acquireIndexLock(dir); + expect(readIndexLock(lockPath)?.pid).toBe(process.pid); + release(); + }); +}); + +describe("error-log", () => { + let dir: string; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it("appendIndexError writes TSV lines", () => { + dir = scratchDir("error-log-"); + appendIndexError(dir, "src/a.ts", "boom"); + const text = readFileSync(errorLogPath(dir), "utf-8"); + expect(text).toContain("\tsrc/a.ts\tboom\n"); + }); +}); diff --git a/src/application/index-lock.ts b/src/application/index-lock.ts new file mode 100644 index 0000000..341423f --- /dev/null +++ b/src/application/index-lock.ts @@ -0,0 +1,140 @@ +import { + existsSync, + mkdirSync, + readFileSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +/** Cross-process index lock filename inside `/`. */ +export const INDEX_LOCK_NAME = "index.lock"; + +/** Default max lock age before auto-steal when the holder PID is still alive. */ +export const DEFAULT_LOCK_MAX_AGE_MS = 4 * 60 * 60 * 1000; + +export interface IndexLockPayload { + pid: number; + started_at: string; +} + +export class IndexLockHeldError extends Error { + constructor(message: string) { + super(message); + this.name = "IndexLockHeldError"; + } +} + +export function indexLockPath(stateDir: string): string { + return join(stateDir, INDEX_LOCK_NAME); +} + +export function readIndexLock(lockPath: string): IndexLockPayload | null { + if (!existsSync(lockPath)) return null; + try { + const data = JSON.parse( + readFileSync(lockPath, "utf-8"), + ) as IndexLockPayload; + if (typeof data.pid !== "number" || typeof data.started_at !== "string") { + return null; + } + return data; + } catch { + return null; + } +} + +export function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export function isStaleLock( + payload: IndexLockPayload, + opts?: { maxAgeMs?: number; now?: number }, +): boolean { + if (!isPidAlive(payload.pid)) return true; + const started = Date.parse(payload.started_at); + if (Number.isNaN(started)) return true; + const maxAge = opts?.maxAgeMs ?? DEFAULT_LOCK_MAX_AGE_MS; + return (opts?.now ?? Date.now()) - started > maxAge; +} + +function releaseIndexLock(lockPath: string, pid: number): void { + try { + const current = readIndexLock(lockPath); + if (current?.pid === pid) unlinkSync(lockPath); + } catch { + // best-effort — stale unlink on next acquire + } +} + +/** + * Acquire `/index.lock`. Throws {@link IndexLockHeldError} when a + * live, non-stale lock exists. Auto-steals stale locks with a stderr warning. + */ +export function acquireIndexLock(stateDir: string): () => void { + mkdirSync(stateDir, { recursive: true }); + const lockPath = indexLockPath(stateDir); + const payload: IndexLockPayload = { + pid: process.pid, + started_at: new Date().toISOString(), + }; + const body = `${JSON.stringify(payload)}\n`; + + const tryWrite = (): void => { + writeFileSync(lockPath, body, { encoding: "utf-8", flag: "wx" }); + }; + + try { + tryWrite(); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw err; + const existing = readIndexLock(lockPath); + if (existing && isStaleLock(existing)) { + // eslint-disable-next-line no-console -- intentional ops warning on stderr + console.error( + `[codemap] removing stale index lock (pid ${existing.pid})`, + ); + unlinkSync(lockPath); + tryWrite(); + return () => releaseIndexLock(lockPath, payload.pid); + } + const holder = existing?.pid ?? "unknown"; + throw new IndexLockHeldError( + `Index already running (lock held by pid ${holder}). Wait for it to finish or run \`codemap unlock\` if the process died.`, + ); + } + + return () => releaseIndexLock(lockPath, payload.pid); +} + +/** + * Remove `index.lock`. Without `--force`, refuses when the holder PID is alive + * and the lock is not stale. + */ +export function removeIndexLock( + stateDir: string, + opts?: { force?: boolean }, +): boolean { + const lockPath = indexLockPath(stateDir); + if (!existsSync(lockPath)) return false; + const existing = readIndexLock(lockPath); + if ( + !opts?.force && + existing && + !isStaleLock(existing) && + isPidAlive(existing.pid) + ) { + throw new IndexLockHeldError( + `Index lock held by live pid ${existing.pid}. Use \`codemap unlock --force\` to remove anyway.`, + ); + } + unlinkSync(lockPath); + return true; +} diff --git a/src/application/run-index.ts b/src/application/run-index.ts index c73688c..c394a5f 100644 --- a/src/application/run-index.ts +++ b/src/application/run-index.ts @@ -7,6 +7,7 @@ import { } from "../db"; import type { CodemapDatabase } from "../db"; import { getBoundaryRules, getFts5Enabled } from "../runtime"; +import { getStateDir } from "../runtime"; import { collectFiles, deleteFilesFromIndex, @@ -16,6 +17,7 @@ import { indexFiles, targetedReindex, } from "./index-engine"; +import { acquireIndexLock } from "./index-lock"; import type { IndexResult, IndexTableStats } from "./types"; /** @@ -114,10 +116,32 @@ export interface RunIndexOptions { * * @remarks * Call `initCodemap()` and `configureResolver()` for this project before invoking (same as CLI bootstrap). + * Serialises in-process and via `/index.lock` for cross-process safety. */ +let indexRunChain: Promise = Promise.resolve(); + export async function runCodemapIndex( db: CodemapDatabase, options: RunIndexOptions = {}, +): Promise { + const run = indexRunChain.then(async () => { + const release = acquireIndexLock(getStateDir()); + try { + return await runCodemapIndexBody(db, options); + } finally { + release(); + } + }); + indexRunChain = run.then( + () => {}, + () => {}, + ); + return run as Promise; +} + +async function runCodemapIndexBody( + db: CodemapDatabase, + options: RunIndexOptions = {}, ): Promise { const quiet = options.quiet ?? false; let mode: IndexMode = options.mode ?? "incremental"; diff --git a/src/application/state-dir.ts b/src/application/state-dir.ts index 8bba4fb..c56409f 100644 --- a/src/application/state-dir.ts +++ b/src/application/state-dir.ts @@ -53,6 +53,8 @@ export const STATE_GITIGNORE_BODY = `# Managed by codemap — overwritten on nex index.db index.db-shm index.db-wal +index.lock +errors.log audit-cache/ `; diff --git a/src/cli/bootstrap.ts b/src/cli/bootstrap.ts index d3d2727..4864f6b 100644 --- a/src/cli/bootstrap.ts +++ b/src/cli/bootstrap.ts @@ -63,6 +63,7 @@ Coverage ingest (Istanbul JSON or LCOV from any test runner): codemap ingest-coverage [--json] # path = file or dir; format auto-detected Other: + codemap unlock [--force] Remove stale cross-process index lock codemap version codemap --version, -V diff --git a/src/cli/cmd-index.ts b/src/cli/cmd-index.ts index 7ef6253..0104709 100644 --- a/src/cli/cmd-index.ts +++ b/src/cli/cmd-index.ts @@ -1,6 +1,7 @@ import { extname } from "node:path"; import { VALID_EXTENSIONS } from "../application/index-engine"; +import { IndexLockHeldError } from "../application/index-lock"; import { runCodemapIndex } from "../application/run-index"; import { closeDb, openDb } from "../db"; import { bootstrapCodemap } from "./bootstrap-codemap"; @@ -38,6 +39,12 @@ export async function runIndexCmd(opts: { performance: reportPerformance, }); } + } catch (err) { + if (err instanceof IndexLockHeldError) { + console.error(err.message); + process.exit(1); + } + throw err; } finally { closeDb(db); } diff --git a/src/cli/cmd-unlock.ts b/src/cli/cmd-unlock.ts new file mode 100644 index 0000000..ed02c92 --- /dev/null +++ b/src/cli/cmd-unlock.ts @@ -0,0 +1,68 @@ +import { IndexLockHeldError, removeIndexLock } from "../application/index-lock"; +import { resolveStateDir } from "../application/state-dir"; + +export function printUnlockCmdHelp(): void { + console.log(`Usage: codemap unlock [--force] + +Remove a stale cross-process index lock (/index.lock). + +Flags: + --force Remove the lock even when the holder PID is still alive + --help, -h Show this help + +When indexing fails with "Index already running", another codemap process +(MCP, watch, git hook, or CLI) may hold the lock. If that process died, +run \`codemap unlock\`. See /errors.log for per-file parse failures. +`); +} + +export function parseUnlockRest( + rest: string[], +): + | { kind: "help" } + | { kind: "error"; message: string } + | { kind: "run"; force: boolean } { + if (rest[0] !== "unlock") { + throw new Error("parseUnlockRest: expected unlock"); + } + let force = false; + for (let i = 1; i < rest.length; i++) { + const a = rest[i]; + if (a === "--help" || a === "-h") return { kind: "help" }; + if (a === "--force") { + force = true; + continue; + } + return { + kind: "error", + message: `codemap: unknown option "${a}". Run codemap unlock --help for usage.`, + }; + } + return { kind: "run", force }; +} + +export async function runUnlockCmd(opts: { + root: string; + stateDir?: string | undefined; + force: boolean; +}): Promise { + const stateDir = resolveStateDir({ + root: opts.root, + cliFlag: opts.stateDir, + env: process.env.CODEMAP_STATE_DIR, + }); + try { + const removed = removeIndexLock(stateDir, { force: opts.force }); + if (removed) { + console.error("Index lock removed."); + } else { + console.error("No index lock present."); + } + } catch (err) { + if (err instanceof IndexLockHeldError) { + console.error(err.message); + process.exit(1); + } + throw err; + } +} diff --git a/src/cli/main.ts b/src/cli/main.ts index 40cdee5..b86c8c3 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -490,6 +490,22 @@ Copies bundled agent templates into .agents/ under the project root. return; } + if (rest[0] === "unlock") { + const { parseUnlockRest, printUnlockCmdHelp, runUnlockCmd } = + await import("./cmd-unlock.js"); + const parsed = parseUnlockRest(rest); + if (parsed.kind === "help") { + printUnlockCmdHelp(); + return; + } + if (parsed.kind === "error") { + console.error(parsed.message); + process.exit(1); + } + await runUnlockCmd({ root, stateDir, force: parsed.force }); + return; + } + const { runIndexCmd } = await import("./cmd-index.js"); await runIndexCmd({ root, configFile, stateDir, fts5Cli, rest }); }