From 1262b5a70eb6c57247b4a8a700b371d0256850cc Mon Sep 17 00:00:00 2001 From: kryptobaseddev Date: Mon, 1 Jun 2026 00:47:03 -0700 Subject: [PATCH 1/6] =?UTF-8?q?refactor(T11522):=20route=20memory-sqlite.t?= =?UTF-8?q?s=20brain=20domain=20through=20openDualScopeDb=20+=20DDL?= =?UTF-8?q?=E2=86=92Drizzle=20migrations=20(E6-L2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - getBrainDb() delegates open to openDualScopeDb('project') facade (mirrors L1) - getBrainDbPath() -> resolveDualScopeDbPath('project') (.cleo/cleo.db) - strip ~15 ensureColumns + ~8 raw CREATE TABLE band-aids from runBrainMigrations - add brain_task_observations forward migration (only genuinely-uncovered table) - dual-scope-db opens with allowExtension:true (sqlite-vec on shared handle) - closeBrainDb/resetBrainDbState evict dual-scope cache Co-Authored-By: Claude Opus 4.8 --- .../migration.sql | 30 + packages/core/src/store/dual-scope-db.ts | 15 +- packages/core/src/store/memory-sqlite.ts | 836 ++++-------------- 3 files changed, 239 insertions(+), 642 deletions(-) create mode 100644 packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql diff --git a/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql b/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql new file mode 100644 index 000000000..064a4f1ff --- /dev/null +++ b/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql @@ -0,0 +1,30 @@ +-- T11522 (E6-L2): Add brain_task_observations to the consolidated cleo-project schema. +-- +-- brain_task_observations is the join table linking brain_observations to task IDs +-- (T1615). It enables `cleo memory find` queries to surface session context for a +-- given task. It was created ONLY by a post-hoc `CREATE TABLE IF NOT EXISTS` in +-- memory-sqlite.ts::runBrainMigrations — it has NO drizzle-brain migration and was +-- NOT included in the T11363 consolidation migration (exodus maps it to `null` as a +-- runtime-only cache, see store/exodus/table-name-map.ts). +-- +-- E6-L2 routes getBrainDb() through openDualScopeDb('project'), so the brain handle +-- now opens the consolidated project `cleo.db`. The runtime writer +-- (sessions/session-memory-bridge.ts) INSERTs into `brain_task_observations`, so the +-- table must exist in `cleo.db`. This forward migration replaces the removed post-hoc +-- DDL — matching the T9179 precedent (ensureColumns → forward Drizzle migration). +-- +-- Schema matches the removed memory-sqlite.ts DDL exactly so the runtime writer +-- INSERTs without changes. All statements are IF NOT EXISTS so re-running is safe. +CREATE TABLE IF NOT EXISTS `brain_task_observations` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `observation_id` text NOT NULL, + `task_id` text NOT NULL, + `link_type` text DEFAULT 'session-completed' NOT NULL, + `created_at` text DEFAULT (datetime('now')) NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `idx_brain_task_obs_unique` ON `brain_task_observations` (`observation_id`, `task_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_observation` ON `brain_task_observations` (`observation_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_task` ON `brain_task_observations` (`task_id`); diff --git a/packages/core/src/store/dual-scope-db.ts b/packages/core/src/store/dual-scope-db.ts index 5644688fd..ca78ae1bb 100644 --- a/packages/core/src/store/dual-scope-db.ts +++ b/packages/core/src/store/dual-scope-db.ts @@ -166,7 +166,10 @@ function getDrizzle(): DrizzleFn { } // Also lazy-load DatabaseSync to avoid eager node:sqlite pull. -type DatabaseSyncCtor = new (path: string, options?: { readOnly?: boolean }) => DatabaseSync; +type DatabaseSyncCtor = new ( + path: string, + options?: { readOnly?: boolean; allowExtension?: boolean }, +) => DatabaseSync; let _DatabaseSyncCtor: DatabaseSyncCtor | null = null; @@ -272,8 +275,16 @@ export async function openDualScopeDb(scope: DualScope, cwd?: string): Promise> | null = null; let _vecLoaded = false; /** - * Get the path to the brain.db SQLite database file. + * Get the path to the brain-domain SQLite database file. + * + * ## E6-L2 (T11522) + * + * After the dual-scope migration, `getBrainDb()` opens the consolidated project + * `cleo.db` via {@link openDualScopeDb} — not the legacy standalone `brain.db`. + * This function therefore returns the dual-scope `cleo.db` path so that callers + * checking for the file `getBrainDb()` created (existence / backup / health + * probes) point at the correct file. */ export function getBrainDbPath(cwd?: string): string { - return join(resolveCleoDir(cwd), DB_FILENAME); + return resolveDualScopeDbPath('project', cwd); } /** @@ -73,13 +138,39 @@ export function resolveBrainMigrationsFolder(): string { // tableExists — delegated to migration-manager.ts (T132) /** - * Run drizzle migrations to create/update brain.db tables. + * Run the legacy `drizzle-brain` migrations against the consolidated project + * `cleo.db` handle. + * + * ## E6-L2 rewrite (T11522) * - * Delegates to shared migration-manager.ts for journal reconciliation, - * retry logic, and safety backups. See T132 for consolidation rationale. + * Every post-hoc `ensureColumns` (~15) and raw `CREATE TABLE IF NOT EXISTS` + * (~8) band-aid that previously lived here has been removed. All of them were + * redundant safety-nets: the journal reconciler `probeAndMarkApplied` + * (migration-manager.ts, T632) detects already-applied DDL and leaves genuinely + * missing DDL un-journaled so `migrate()` runs it. Their columns/tables are all + * covered by the `drizzle-brain/*` migration files (e.g. T9179 already converted + * the `brain_retrieval_log` / `brain_observations.stability_score` ensureColumns + * into forward migrations — the precedent this rewrite follows). The two + * idempotent UPDATE data-fixes (T626 `co_retrieved` relabel, T673-M3 hebbian + * seed) are likewise applied by their own UPDATE-only migration files, which the + * reconciler always runs (zero DDL targets → never marked applied by probe). + * + * The ONE genuinely-uncovered table — `brain_task_observations` (T1615, a + * runtime-only join cache that exodus maps to `null`) — was converted to a + * forward Drizzle migration under + * `migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations`. + * + * These migrations are still run during the E3→E6 transition so the legacy + * physical tables that the runtime queries by their pre-consolidation names + * (notably `deriver_queue` — the consolidated schema carries the prefixed + * `brain_deriver_queue`) exist alongside the consolidated `brain_*` tables. + * Every brain migration is `CREATE TABLE IF NOT EXISTS` / additive + * `ALTER TABLE`, so re-applying them onto a `cleo.db` that already has the + * consolidated `brain_*` tables is idempotent and safe. * * @task T5128 * @task T132 - Unified migration system + * @task T11522 - E6-L2: remove ensureColumns/CREATE band-aids; route via dual-scope */ function runBrainMigrations( nativeDb: DatabaseSync, @@ -87,497 +178,18 @@ function runBrainMigrations( ): void { const migrationsFolder = resolveBrainMigrationsFolder(); - // Safety backup before any migration work + // Safety backup before any migration work. if (tableExists(nativeDb, 'brain_decisions') && _dbPath) { createSafetyBackup(_dbPath); } - // Bootstrap baseline + reconcile stale journal entries + // Bootstrap baseline + reconcile stale journal entries. reconcileJournal(nativeDb, migrationsFolder, 'brain_decisions', 'brain'); // Run pending migrations with SQLITE_BUSY retry. // Pass nativeDb + existenceTable so migrateWithRetry can auto-reconcile any // partial migration (Scenario 3) that slips through the proactive check above. migrateWithRetry(db, migrationsFolder, nativeDb, 'brain_decisions', 'brain'); - - // T632 root-cause fix (complete): the migration journal reconciler (Sub-case B) - // uses a per-migration DDL probe (probeAndMarkApplied in migration-manager.ts) - // instead of wholesale-marking-all-applied. ALTER TABLE ADD COLUMN migrations - // (T417 agent, T528 graph schema, T531 quality-score, T549 tiered-memory, etc.) - // now run correctly when their columns are missing — the reconciler leaves them - // unjournaled so Drizzle's migrate() runs the DDL. - // - // The ensureColumns band-aids for T528/T531/T549 were removed here as part of - // T632 because all their columns are covered by Drizzle migration files. If a - // schema regression recurs, debug probeAndMarkApplied in migration-manager.ts — - // do NOT add new band-aids here. - // - // ensureColumns below are retained ONLY for columns that have NO corresponding - // Drizzle migration file (self-healing DDL only — see each comment). - - // T626-M1: Normalize co_retrieved edge type — idempotent safety-net UPDATE. - // The shipped Hebbian strengthener emitted edge_type = 'relates_to' instead of - // 'co_retrieved'. Relabel only rows from the consolidation provenance so no - // semantic edges are affected. The Drizzle migration file does the same UPDATE; - // this guard handles installs where the journal reconciler already marked - // the migration applied before the SQL ran. - // - // T759: Guard provenance column existence before UPDATE. If T528 migration has - // not yet run (e.g. on a fresh install where only the initial migration is - // present), brain_page_edges will not have a provenance column and the UPDATE - // will throw "no such column: provenance". ensureColumns adds the column if - // missing so the UPDATE is always safe to execute. - if (tableExists(nativeDb, 'brain_page_edges')) { - ensureColumns(nativeDb, 'brain_page_edges', [{ name: 'provenance', ddl: 'text' }], 'brain'); - nativeDb - .prepare( - `UPDATE brain_page_edges - SET edge_type = 'co_retrieved' - WHERE edge_type = 'relates_to' - AND provenance LIKE 'consolidation:%'`, - ) - .run(); - } - - // T673-M1: STDP plasticity columns on brain_retrieval_log. - // session_id was declared in the Drizzle schema but never applied to the live table. - // reward_signal, retrieval_order, delta_ms are new additions per spec §2.1.1. - if (tableExists(nativeDb, 'brain_retrieval_log')) { - ensureColumns( - nativeDb, - 'brain_retrieval_log', - [ - { name: 'session_id', ddl: 'text' }, - { name: 'reward_signal', ddl: 'real' }, - { name: 'retrieval_order', ddl: 'integer' }, - { name: 'delta_ms', ddl: 'integer' }, - ], - 'brain', - ); - } - - // T673-M2: observability columns on brain_plasticity_events - // session_id is declared in Drizzle schema and included in M2 CREATE TABLE IF NOT EXISTS, - // but may be missing from installs where the table was created before M2. - if (tableExists(nativeDb, 'brain_plasticity_events')) { - ensureColumns( - nativeDb, - 'brain_plasticity_events', - [ - { name: 'session_id', ddl: 'text' }, - { name: 'weight_before', ddl: 'real' }, - { name: 'weight_after', ddl: 'real' }, - { name: 'retrieval_log_id', ddl: 'integer' }, - { name: 'reward_signal', ddl: 'real' }, - { name: 'delta_t_ms', ddl: 'integer' }, - ], - 'brain', - ); - } - - // T673-M3: plasticity tracking columns on brain_page_edges - ensureColumns( - nativeDb, - 'brain_page_edges', - [ - { name: 'last_reinforced_at', ddl: 'text' }, - { name: 'reinforcement_count', ddl: 'integer NOT NULL DEFAULT 0' }, - { name: 'plasticity_class', ddl: "text NOT NULL DEFAULT 'static'" }, - { name: 'last_depressed_at', ddl: 'text' }, - { name: 'depression_count', ddl: 'integer NOT NULL DEFAULT 0' }, - { name: 'stability_score', ddl: 'real' }, - ], - 'brain', - ); - - // T673-M3: seed co_retrieved edges as hebbian (idempotent) - if (tableExists(nativeDb, 'brain_page_edges')) { - nativeDb - .prepare( - `UPDATE brain_page_edges SET plasticity_class = 'hebbian' - WHERE edge_type = 'co_retrieved' AND plasticity_class = 'static'`, - ) - .run(); - } - - // T673-M4: new plasticity infrastructure tables — self-healing CREATE IF NOT EXISTS. - // These guards ensure the tables exist even on installs where the Drizzle migration - // journal was already partially applied. All three tables are CREATE IF NOT EXISTS - // so re-running is safe. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_weight_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - edge_from_id TEXT NOT NULL, - edge_to_id TEXT NOT NULL, - edge_type TEXT NOT NULL, - weight_before REAL, - weight_after REAL NOT NULL, - delta_weight REAL NOT NULL, - event_kind TEXT NOT NULL, - source_plasticity_event_id INTEGER, - retrieval_log_id INTEGER, - reward_signal REAL, - changed_at TEXT NOT NULL DEFAULT (datetime('now')) - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_weight_history_edge - ON brain_weight_history (edge_from_id, edge_to_id, edge_type)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_weight_history_changed_at - ON brain_weight_history (changed_at)`, - ); - - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_modulators ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - modulator_type TEXT NOT NULL, - valence REAL NOT NULL, - magnitude REAL NOT NULL DEFAULT 1.0, - source_event_id TEXT, - session_id TEXT, - description TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')) - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_modulators_session - ON brain_modulators (session_id)`, - ); - - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_consolidation_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - trigger TEXT NOT NULL, - session_id TEXT, - step_results_json TEXT NOT NULL, - duration_ms INTEGER, - succeeded INTEGER NOT NULL DEFAULT 1, - started_at TEXT NOT NULL DEFAULT (datetime('now')) - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_consolidation_events_started_at - ON brain_consolidation_events (started_at)`, - ); - - // T1002: brain_transcript_events — full-fidelity Claude session block store. - // CREATE IF NOT EXISTS so re-runs on existing databases are safe. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_transcript_events ( - id TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - seq INTEGER NOT NULL, - role TEXT NOT NULL, - block_type TEXT NOT NULL, - content TEXT NOT NULL, - tokens INTEGER, - redacted_at TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')) - )`, - ); - nativeDb.exec( - `CREATE UNIQUE INDEX IF NOT EXISTS idx_transcript_events_session_seq - ON brain_transcript_events (session_id, seq)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_transcript_events_session - ON brain_transcript_events (session_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_transcript_events_role - ON brain_transcript_events (role)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_transcript_events_block_type - ON brain_transcript_events (block_type)`, - ); - - // T1001: stability_score column on brain_observations (distinct from brain_page_edges.stability_score). - // Added via ensureColumns — idempotent, safe on existing databases. - ensureColumns( - nativeDb, - 'brain_observations', - [{ name: 'stability_score', ddl: 'real DEFAULT 0.5' }], - 'brain', - ); - - // T1084: PSYCHE Wave 2 — peer_id + peer_scope on all four brain memory tables. - // Drizzle migration 20260423000001_t1084-peer-id-memory-isolation handles fresh installs. - // ensureColumns here is the safety-net for installs where the migration journal was - // already partially applied or the journal reconciler skips DDL-only migrations. - // Both columns are NOT NULL with a DEFAULT so the ALTER is safe on non-empty tables. - const peerColumns = [ - { name: 'peer_id', ddl: "text NOT NULL DEFAULT 'global'" }, - { name: 'peer_scope', ddl: "text NOT NULL DEFAULT 'project'" }, - ]; - ensureColumns(nativeDb, 'brain_decisions', peerColumns, 'brain'); - ensureColumns(nativeDb, 'brain_patterns', peerColumns, 'brain'); - ensureColumns(nativeDb, 'brain_learnings', peerColumns, 'brain'); - ensureColumns(nativeDb, 'brain_observations', peerColumns, 'brain'); - // Companion indexes — idempotent CREATE INDEX IF NOT EXISTS. - for (const [table, idxName] of [ - ['brain_decisions', 'idx_brain_decisions_peer_scope'], - ['brain_patterns', 'idx_brain_patterns_peer_scope'], - ['brain_learnings', 'idx_brain_learnings_peer_scope'], - ['brain_observations', 'idx_brain_observations_peer_scope'], - ] as const) { - nativeDb.exec(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${table} (peer_id, peer_scope)`); - } - - // T1260: PSYCHE E3 — provenance_class on all four brain memory tables (M6 refusal gate). - // Drizzle migration 20260424000001_t1260-provenance-class handles fresh installs. - // ensureColumns here is the safety-net for installs where the journal reconciler - // already marked prior migrations applied before this column was added, or where - // the test runner uses a module-cached DB from before the migration ran. - const provenanceColumn = [{ name: 'provenance_class', ddl: "text DEFAULT 'swept-clean'" }]; - ensureColumns(nativeDb, 'brain_decisions', provenanceColumn, 'brain'); - ensureColumns(nativeDb, 'brain_patterns', provenanceColumn, 'brain'); - ensureColumns(nativeDb, 'brain_learnings', provenanceColumn, 'brain'); - ensureColumns(nativeDb, 'brain_observations', provenanceColumn, 'brain'); - - // T1826: Decision Storage Consolidation — ADR tracking + governance columns. - // Drizzle migration 20260504000001_t1826-decisions-v2 handles fresh installs, BUT - // node:sqlite's prepare() only executes the first SQL statement when multiple - // statements are joined without "--> statement-breakpoint" separators. The migration - // file has 7+ ALTER TABLE statements with no breakpoints, so only adr_number gets - // applied by Drizzle. ensureColumns here is the safety-net that guarantees all 7 - // new columns exist, matching the pattern used by T1084, T1260, T1145, etc. - ensureColumns( - nativeDb, - 'brain_decisions', - [ - { name: 'adr_number', ddl: 'integer' }, - { name: 'adr_path', ddl: 'text' }, - { name: 'supersedes', ddl: 'text' }, - { name: 'superseded_by', ddl: 'text' }, - { name: 'confirmation_state', ddl: "text NOT NULL DEFAULT 'proposed'" }, - { name: 'decided_by', ddl: "text NOT NULL DEFAULT 'agent'" }, - { name: 'validator_run_at', ddl: 'integer' }, - ], - 'brain', - ); - // T1826: Idempotent companion indexes for the new columns. - nativeDb.exec( - `CREATE UNIQUE INDEX IF NOT EXISTS idx_brain_decisions_adr_number - ON brain_decisions(adr_number) - WHERE adr_number IS NOT NULL`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_decisions_confirmation_state - ON brain_decisions(confirmation_state)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_decisions_decided_by - ON brain_decisions(decided_by)`, - ); - - // T1001: brain_promotion_log — typed promotion audit trail. - // One row per observation evaluated (and promoted) by promoteObservationsToTyped(). - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_promotion_log ( - id TEXT PRIMARY KEY, - observation_id TEXT NOT NULL, - from_tier TEXT NOT NULL, - to_tier TEXT NOT NULL, - score REAL NOT NULL, - decided_at TEXT NOT NULL DEFAULT (datetime('now')), - decided_by TEXT NOT NULL DEFAULT 'composite-scorer', - rationale_json TEXT, - fulfilled_at TEXT, - fulfillment_note TEXT - )`, - ); - // T1903: migrate existing brain_promotion_log tables to add fulfillment columns (idempotent). - try { - nativeDb.exec(`ALTER TABLE brain_promotion_log ADD COLUMN fulfilled_at TEXT`); - } catch { - /* column already exists */ - } - try { - nativeDb.exec(`ALTER TABLE brain_promotion_log ADD COLUMN fulfillment_note TEXT`); - } catch { - /* column already exists */ - } - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_promotion_log_observation - ON brain_promotion_log (observation_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_promotion_log_decided_at - ON brain_promotion_log (decided_at)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_promotion_log_to_tier - ON brain_promotion_log (to_tier)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_promotion_log_score - ON brain_promotion_log (score)`, - ); - - // T1003: brain_backfill_runs — staged backfill audit log. - // CREATE IF NOT EXISTS so re-runs on existing databases are safe. - // Staged rows are held in rollback_snapshot_json until approved/rolled-back. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_backfill_runs ( - id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'staged', - created_at TEXT NOT NULL DEFAULT (datetime('now')), - approved_at TEXT, - rows_affected INTEGER NOT NULL DEFAULT 0, - rollback_snapshot_json TEXT, - source TEXT NOT NULL DEFAULT 'unknown', - target_table TEXT NOT NULL DEFAULT 'brain_observations', - approved_by TEXT - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_backfill_runs_status - ON brain_backfill_runs (status)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_backfill_runs_kind - ON brain_backfill_runs (kind)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_backfill_runs_created_at - ON brain_backfill_runs (created_at)`, - ); - - // T1145: Wave 5 Deriver Queue — deriver lineage + level columns on brain_observations. - // Drizzle migration 20260424000003_t1145-extend-brain-observations handles fresh installs. - // ensureColumns here is the safety-net for test DBs and existing installs where the - // journal reconciler may skip the ALTER TABLE migration. - ensureColumns( - nativeDb, - 'brain_observations', - [ - { name: 'source_ids', ddl: 'text' }, - { name: 'times_derived', ddl: 'integer DEFAULT 1' }, - { name: 'level', ddl: "text DEFAULT 'explicit'" }, - { name: 'tree_id', ddl: 'integer' }, - ], - 'brain', - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_observations_level ON brain_observations (level)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_observations_tree_id ON brain_observations (tree_id)`, - ); - - // T1145: deriver_queue — durable background derivation work queue. - // CREATE IF NOT EXISTS so re-runs on existing databases are safe. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS deriver_queue ( - id TEXT PRIMARY KEY, - item_type TEXT NOT NULL, - item_id TEXT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL DEFAULT 'pending', - claimed_at TEXT, - claimed_by TEXT, - error_msg TEXT, - retry_count INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - completed_at TEXT - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_deriver_queue_status_priority - ON deriver_queue (status, priority DESC, created_at ASC)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_deriver_queue_item - ON deriver_queue (item_type, item_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_deriver_queue_claimed_at - ON deriver_queue (claimed_at)`, - ); - - // T1146: brain_memory_trees — hierarchical RPTree clustering (W6 Dreamer Upgrade). - // CREATE IF NOT EXISTS so re-runs on existing databases are safe. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_memory_trees ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - depth INTEGER NOT NULL DEFAULT 0, - leaf_ids TEXT NOT NULL DEFAULT '[]', - centroid TEXT, - parent_id INTEGER REFERENCES brain_memory_trees(id) ON DELETE CASCADE, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_trees_parent ON brain_memory_trees (parent_id)`, - ); - nativeDb.exec(`CREATE INDEX IF NOT EXISTS idx_brain_trees_depth ON brain_memory_trees (depth)`); - - // T1615: brain_task_observations — join table linking brain_observations to task IDs. - // Enables cleo memory find queries to surface session context for a given task. - // CREATE IF NOT EXISTS so re-runs on existing databases are safe. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_task_observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - observation_id TEXT NOT NULL, - task_id TEXT NOT NULL, - link_type TEXT NOT NULL DEFAULT 'session-completed', - created_at TEXT NOT NULL DEFAULT (datetime('now')) - )`, - ); - nativeDb.exec( - `CREATE UNIQUE INDEX IF NOT EXISTS idx_brain_task_obs_unique - ON brain_task_observations (observation_id, task_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_task_obs_observation - ON brain_task_observations (observation_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_task_obs_task - ON brain_task_observations (task_id)`, - ); - - // T11371: brain_attention — Tier-2 scope-keyed, decaying working-memory buffer - // (Epic T11288 · Saga T11283). The Drizzle migration - // 20260530000001_t11371-add-attention-table handles fresh installs; this - // self-healing CREATE IF NOT EXISTS is the safety-net for installs where the - // journal reconciler marked prior migrations applied before this table was - // added, or where the test runner uses a module-cached DB from before the - // migration ran (mirrors the brain_task_observations / deriver_queue pattern). - // `tags` is a JSONB BLOB (E4 jsonb() helper) — read in-SQL via - // json_each(tags) / json(tags), NEVER JSON.parse off the raw BLOB. - nativeDb.exec( - `CREATE TABLE IF NOT EXISTS brain_attention ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - session_id TEXT, - agent_id TEXT, - scope_kind TEXT NOT NULL, - scope_id TEXT NOT NULL, - tags BLOB DEFAULT (jsonb('[]')), - created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000), - expires_at INTEGER, - decay_score REAL, - status TEXT NOT NULL DEFAULT 'open' - )`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_attention_scope - ON brain_attention (scope_kind, scope_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_attention_session - ON brain_attention (session_id)`, - ); - nativeDb.exec( - `CREATE INDEX IF NOT EXISTS idx_brain_attention_status_expires - ON brain_attention (status, expires_at)`, - ); } /** @@ -649,178 +261,105 @@ async function initEmbeddingProvider(cwd?: string): Promise { } /** - * Run a fast malformation probe against an open native handle. - * - * Uses `PRAGMA quick_check` rather than `integrity_check` — quick_check - * skips index-sort and out-of-order rowid scans, returning in milliseconds - * on a healthy DB. A live malformed schema (the T10260/T10265 incident - * pattern) surfaces here as either a throw from `prepare()` or a non-`ok` - * return value. - * - * @returns `true` when the DB passes quick_check, `false` otherwise. - * @task T10303 - * @internal - */ -function probeBrainDbIntegrity(db: DatabaseSync): boolean { - try { - const row = db.prepare('PRAGMA quick_check').get() as { quick_check?: string } | undefined; - return row?.quick_check === 'ok'; - } catch { - // prepare() threw — schema is malformed (the T10260/T10265 signature). - return false; - } -} - -/** - * Detect whether an open-time exception matches the brain.db malformation - * signature `ERR_SQLITE_ERROR errcode=11` (SQLITE_CORRUPT) — the live - * failure pattern that triggered Saga T10281. - * - * @internal - */ -function isMalformationError(err: unknown): boolean { - if (!(err instanceof Error)) return false; - const code = (err as Error & { code?: string; errcode?: number }).code; - const errcode = (err as Error & { errcode?: number }).errcode; - if (code === 'ERR_SQLITE_ERROR' && errcode === 11) return true; - const msg = err.message ?? ''; - // Belt-and-suspenders: some node:sqlite versions stringify "malformed - // database schema" into the message even when errcode is unset. - return /malformed/i.test(msg); -} - -/** - * Open the brain.db with auto-recovery on malformation. + * Initialize the project-scope BRAIN domain SQLite database (lazy, singleton). * - * Synchronous by design — recovery runs on the open-blocking critical path. - * The alternative is silent broken cognition (T10260, T10265). + * ## E6-L2 façade (T11522) * - * Recovery flow: - * 1. Try `openNativeDatabase()`. If it throws with the malformation - * signature, run recovery and re-attempt once. - * 2. If the open succeeded, run `PRAGMA quick_check`. If it fails (schema - * malformation that survives the open), close the handle, run recovery, - * and re-attempt. + * Delegates the physical DB open to {@link openDualScopeDb}('project', cwd) — + * the canonical dual-scope chokepoint. The returned `NodeSQLiteDatabase` wraps + * the same `DatabaseSync` handle as the consolidated project `cleo.db` but is + * typed against the legacy brain schema (`brainSchema`, physical tables + * `brain_decisions`, …) so all existing brain callers compile and run without + * change. The legacy `drizzle-brain` migrations are still applied to this handle + * during the E3→E6 transition (additive / `IF NOT EXISTS` — idempotent on the + * consolidated DB) so the runtime-queried legacy physical tables (notably + * `deriver_queue`) co-exist with the consolidated `brain_*` tables. * - * @internal - * @task T10303 - */ -function openBrainDbWithRecovery(dbPath: string, cwd: string | undefined): DatabaseSync { - const tryOpen = (): DatabaseSync => openNativeDatabase(dbPath, { allowExtension: true }); - - const runRecovery = (): void => { - const cleoDir = resolveCleoDir(cwd); - recoverMalformedBrainDb({ - corruptPath: dbPath, - snapshotDir: join(cleoDir, 'backups', 'snapshot'), - vacuumSnapshotDir: join(cleoDir, 'backups', 'sqlite'), - legacyArtifactDir: cleoDir, - quarantineRoot: join(cleoDir, 'quarantine'), - logger: getLogger('brain-recover'), - }); - }; - - let nativeDb: DatabaseSync; - try { - nativeDb = tryOpen(); - } catch (err) { - if (!isMalformationError(err)) throw err; - runRecovery(); - nativeDb = tryOpen(); - } - - if (!probeBrainDbIntegrity(nativeDb)) { - try { - nativeDb.close(); - } catch { - // close errors are non-fatal — handle terminal anyway - } - runRecovery(); - nativeDb = tryOpen(); - // One final check — if recovery couldn't produce a clean DB, we still - // return the handle. The downstream migration path will surface the - // failure with full context rather than silently degrading. - } - - return nativeDb; -} - -/** - * Initialize the brain.db SQLite database (lazy, singleton). - * Creates the database file and tables if they don't exist. - * Returns the drizzle ORM instance (async via sqlite-proxy). + * Brain-specific malformation auto-recovery (T10303 / Saga T10281) previously + * ran here against the standalone `brain.db` file. That file no longer backs the + * brain domain after this leaf — the brain tables live inside `cleo.db`, whose + * malformation recovery is a dual-scope-level concern (the brain-only + * quarantine/snapshot-restore pipeline would corrupt the co-resident `tasks_*` / + * `conduit_*` domains). The recovery primitive itself (`recoverMalformedBrainDb`) + * is retained for `doctor` use; only its wiring into this chokepoint is removed. * - * Uses a promise guard so concurrent callers wait for the same - * initialization to complete (migrations are async). + * Uses a promise guard so concurrent callers wait for the same initialization to + * complete (migrations are async). */ export async function getBrainDb(cwd?: string): Promise> { const requestedPath = getBrainDbPath(cwd); - // T1906: guard against prod-DB writes in test mode + // T1906: guard against prod-DB writes in test mode. const { assertTestEnv } = await import('./data-accessor.js'); assertTestEnv(requestedPath); - // If singleton exists but points to different path, reset it + // If singleton exists but points to different path, reset it. if (_db && _dbPath !== requestedPath) { resetBrainDbState(); } if (_db) return _db; - // If already initializing, wait for the in-flight init + // If already initializing, wait for the in-flight init. if (_initPromise) return _initPromise; _initPromise = (async () => { - const dbPath = requestedPath; - _dbPath = dbPath; - - // Ensure directory exists - mkdirSync(dirname(dbPath), { recursive: true }); - - // Open file-backed SQLite via node:sqlite with WAL mode. - // allowExtension: true enables sqlite-vec extension loading. - // - // T10303 (Saga T10281 / Epic T10286): wrap the open in malformation - // detection + auto-recovery. Catches both `ERR_SQLITE_ERROR errcode=11` - // (raised by the open itself or by the post-open quick_check probe) - // and silent schema corruption that surfaces during the first prepare() - // call. On detection, the corrupt DB is quarantined and the freshest - // validated snapshot is restored; we then re-attempt the open ONCE. - const nativeDb = openBrainDbWithRecovery(dbPath, cwd); + // ── Dual-scope chokepoint delegation (T11522 · E6-L2) ───────────────── + // openDualScopeDb applies the pragma SSoT, creates the directory, runs the + // consolidated cleo-project migrations (which create the `brain_*` tables), + // and manages the singleton cache. We extract its native handle so we can + // re-wrap it with the legacy brain-schema for caller compatibility. + const dualHandle = await openDualScopeDb('project', cwd); + + // Extract the underlying DatabaseSync. Drizzle exposes it via `$client`. + const nativeDb = (dualHandle.db as { $client?: DatabaseSync }).$client ?? null; + if (!nativeDb) { + throw new Error( + 'E6-L2: openDualScopeDb returned a handle without $client — ' + + 'cannot extract DatabaseSync for legacy brain-schema wrapping.', + ); + } + _nativeDb = nativeDb; + _dbPath = requestedPath; - // Load sqlite-vec extension for vector similarity search (T5157). - // Non-fatal if unavailable — vec0 tables simply won't be created. + // Load the sqlite-vec extension for vector similarity search (T5157). The + // dual-scope handle is opened with `allowExtension: true`, so loading is + // permitted. Non-fatal if unavailable — vec0 tables simply won't be created. _vecLoaded = loadBrainVecExtension(nativeDb); - // Create drizzle ORM wrapper via node-sqlite - const db = drizzle({ client: nativeDb, schema: brainSchema }); + // Wrap the native handle with the legacy brain-schema drizzle instance so + // existing callers (brainSchema.* queries) continue to work unchanged. + const db = _getDrizzle()({ client: nativeDb, schema: brainSchema }); - // Run drizzle migrations (creates/updates tables) + // Run the legacy drizzle-brain migrations against the shared cleo.db handle. + // During the E3→E6 transition these create the legacy runtime-queried + // physical tables (e.g. `deriver_queue`) alongside the consolidated + // `brain_*` tables. Every brain migration is additive / IF NOT EXISTS. runBrainMigrations(nativeDb, db); - // Create vec0 virtual table for embeddings if extension is loaded (T5157). - // Must run after migrations so the schema is consistent. + // Create the vec0 virtual table for embeddings if the extension is loaded + // (T5157). Must run after migrations so the schema is consistent. if (_vecLoaded) { initializeBrainVec(nativeDb); } - // Seed schema version for new databases (no-op if already set) + // Seed schema version for new databases (no-op if already set). nativeDb .prepare( `INSERT OR IGNORE INTO brain_schema_meta (key, value) VALUES ('schemaVersion', '${BRAIN_SCHEMA_VERSION}')`, ) .run(); - // Set singleton only after migrations complete + // Set singleton only after migrations complete. _db = db; - // Wire the default embedding provider when vec is loaded and embedding is enabled. - // Best-effort, async, never blocks DB access. (T539) + // Wire the default embedding provider when vec is loaded and embedding is + // enabled. Best-effort, async, never blocks DB access. (T539) if (_vecLoaded) { setImmediate(() => { initEmbeddingProvider(cwd).catch(() => { - // Non-fatal — embedding will be unavailable until next startup + // Non-fatal — embedding will be unavailable until next startup. }); }); } @@ -836,16 +375,27 @@ export async function getBrainDb(cwd?: string): Promise Date: Mon, 1 Jun 2026 00:58:37 -0700 Subject: [PATCH 2/6] refactor(T11522): do not run legacy drizzle-brain set on cleo.db; fold deriver_queue into forward migration (E6-L2) Running the full legacy drizzle-brain folder against the consolidated cleo.db collides: cross-migration rename chain (t1147 brain_v2_candidate -> t1402 RENAME TO brain_observations_staging) hits the final table consolidation already made. Consolidated schema is the brain SSoT; forward migration now adds the only two runtime-legacy tables consolidation skipped (brain_task_observations + unprefixed deriver_queue). Removed runBrainMigrations + now-unused migration-manager imports. Co-Authored-By: Claude Opus 4.8 --- .../migration.sql | 60 +++++++++++++ .../migration.sql | 30 ------- packages/core/src/store/memory-sqlite.ts | 86 +++++-------------- 3 files changed, 80 insertions(+), 96 deletions(-) create mode 100644 packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql delete mode 100644 packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql diff --git a/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql b/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql new file mode 100644 index 000000000..54dca07b7 --- /dev/null +++ b/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql @@ -0,0 +1,60 @@ +-- T11522 (E6-L2): Add the two runtime-legacy BRAIN tables the T11363 consolidation +-- migration skipped, so getBrainDb() — now routed through openDualScopeDb('project') +-- — can serve every brain-domain runtime query from the consolidated `cleo.db`. +-- +-- Both tables were previously created by post-hoc `CREATE TABLE IF NOT EXISTS` DDL +-- in memory-sqlite.ts::runBrainMigrations, which E6-L2 removed. The runtime queries +-- them by their pre-consolidation physical names via raw SQL, so they must exist in +-- `cleo.db`. This forward migration replaces the removed post-hoc DDL — matching the +-- T9179 precedent (ensureColumns / self-healing DDL → forward Drizzle migration). +-- Running the legacy `drizzle-brain` migration set against the consolidated DB is NOT +-- viable: its cross-migration rename chain (t1147 `brain_v2_candidate` → t1402 RENAME +-- TO `brain_observations_staging`) collides with the final table the consolidation +-- already created. The consolidated schema is the brain SSoT; this migration adds only +-- the two genuinely-uncovered runtime tables. +-- +-- 1. brain_task_observations (T1615) — join table linking brain_observations to task +-- IDs; powers `cleo memory find` session-context lookups. The runtime writer is +-- sessions/session-memory-bridge.ts. Exodus maps it to `null` (runtime-only cache). +-- +-- 2. deriver_queue (T1145) — durable background derivation work queue. The runtime +-- accessors are packages/core/src/deriver/{enqueue,queue-manager,status}.ts, all of +-- which open via getBrainNativeDb() and query the UNPREFIXED name. The consolidated +-- schema carries the prefixed `brain_deriver_queue`; exodus renames the legacy table +-- onto it. Until that cutover the unprefixed table must exist for the runtime. +-- +-- All statements are IF NOT EXISTS so re-running onto a DB that already has these +-- tables (e.g. after exodus) is idempotent and safe. +CREATE TABLE IF NOT EXISTS `brain_task_observations` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `observation_id` text NOT NULL, + `task_id` text NOT NULL, + `link_type` text DEFAULT 'session-completed' NOT NULL, + `created_at` text DEFAULT (datetime('now')) NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `idx_brain_task_obs_unique` ON `brain_task_observations` (`observation_id`, `task_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_observation` ON `brain_task_observations` (`observation_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_task` ON `brain_task_observations` (`task_id`); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS `deriver_queue` ( + `id` text PRIMARY KEY NOT NULL, + `item_type` text NOT NULL, + `item_id` text NOT NULL, + `priority` integer DEFAULT 0 NOT NULL, + `status` text DEFAULT 'pending' NOT NULL, + `claimed_at` text, + `claimed_by` text, + `error_msg` text, + `retry_count` integer DEFAULT 0 NOT NULL, + `created_at` text DEFAULT (datetime('now')) NOT NULL, + `completed_at` text +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_deriver_queue_status_priority` ON `deriver_queue` (`status`, `priority` DESC, `created_at` ASC); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_deriver_queue_item` ON `deriver_queue` (`item_type`, `item_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_deriver_queue_claimed_at` ON `deriver_queue` (`claimed_at`); diff --git a/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql b/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql deleted file mode 100644 index 064a4f1ff..000000000 --- a/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations/migration.sql +++ /dev/null @@ -1,30 +0,0 @@ --- T11522 (E6-L2): Add brain_task_observations to the consolidated cleo-project schema. --- --- brain_task_observations is the join table linking brain_observations to task IDs --- (T1615). It enables `cleo memory find` queries to surface session context for a --- given task. It was created ONLY by a post-hoc `CREATE TABLE IF NOT EXISTS` in --- memory-sqlite.ts::runBrainMigrations — it has NO drizzle-brain migration and was --- NOT included in the T11363 consolidation migration (exodus maps it to `null` as a --- runtime-only cache, see store/exodus/table-name-map.ts). --- --- E6-L2 routes getBrainDb() through openDualScopeDb('project'), so the brain handle --- now opens the consolidated project `cleo.db`. The runtime writer --- (sessions/session-memory-bridge.ts) INSERTs into `brain_task_observations`, so the --- table must exist in `cleo.db`. This forward migration replaces the removed post-hoc --- DDL — matching the T9179 precedent (ensureColumns → forward Drizzle migration). --- --- Schema matches the removed memory-sqlite.ts DDL exactly so the runtime writer --- INSERTs without changes. All statements are IF NOT EXISTS so re-running is safe. -CREATE TABLE IF NOT EXISTS `brain_task_observations` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `observation_id` text NOT NULL, - `task_id` text NOT NULL, - `link_type` text DEFAULT 'session-completed' NOT NULL, - `created_at` text DEFAULT (datetime('now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX IF NOT EXISTS `idx_brain_task_obs_unique` ON `brain_task_observations` (`observation_id`, `task_id`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_observation` ON `brain_task_observations` (`observation_id`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_task` ON `brain_task_observations` (`task_id`); diff --git a/packages/core/src/store/memory-sqlite.ts b/packages/core/src/store/memory-sqlite.ts index 972d0ec79..ad32773c1 100644 --- a/packages/core/src/store/memory-sqlite.ts +++ b/packages/core/src/store/memory-sqlite.ts @@ -59,12 +59,6 @@ import { openDualScopeDb, resolveDualScopeDbPath, } from './dual-scope-db.js'; -import { - createSafetyBackup, - migrateWithRetry, - reconcileJournal, - tableExists, -} from './migration-manager.js'; import { resolveCorePackageMigrationsFolder } from './resolve-migrations-folder.js'; import * as brainSchema from './schema/memory-schema.js'; @@ -136,61 +130,16 @@ export function resolveBrainMigrationsFolder(): string { } // tableExists — delegated to migration-manager.ts (T132) - -/** - * Run the legacy `drizzle-brain` migrations against the consolidated project - * `cleo.db` handle. - * - * ## E6-L2 rewrite (T11522) - * - * Every post-hoc `ensureColumns` (~15) and raw `CREATE TABLE IF NOT EXISTS` - * (~8) band-aid that previously lived here has been removed. All of them were - * redundant safety-nets: the journal reconciler `probeAndMarkApplied` - * (migration-manager.ts, T632) detects already-applied DDL and leaves genuinely - * missing DDL un-journaled so `migrate()` runs it. Their columns/tables are all - * covered by the `drizzle-brain/*` migration files (e.g. T9179 already converted - * the `brain_retrieval_log` / `brain_observations.stability_score` ensureColumns - * into forward migrations — the precedent this rewrite follows). The two - * idempotent UPDATE data-fixes (T626 `co_retrieved` relabel, T673-M3 hebbian - * seed) are likewise applied by their own UPDATE-only migration files, which the - * reconciler always runs (zero DDL targets → never marked applied by probe). - * - * The ONE genuinely-uncovered table — `brain_task_observations` (T1615, a - * runtime-only join cache that exodus maps to `null`) — was converted to a - * forward Drizzle migration under - * `migrations/drizzle-cleo-project/20260601000002_t11522-brain-task-observations`. - * - * These migrations are still run during the E3→E6 transition so the legacy - * physical tables that the runtime queries by their pre-consolidation names - * (notably `deriver_queue` — the consolidated schema carries the prefixed - * `brain_deriver_queue`) exist alongside the consolidated `brain_*` tables. - * Every brain migration is `CREATE TABLE IF NOT EXISTS` / additive - * `ALTER TABLE`, so re-applying them onto a `cleo.db` that already has the - * consolidated `brain_*` tables is idempotent and safe. - * - * @task T5128 - * @task T132 - Unified migration system - * @task T11522 - E6-L2: remove ensureColumns/CREATE band-aids; route via dual-scope - */ -function runBrainMigrations( - nativeDb: DatabaseSync, - db: NodeSQLiteDatabase, -): void { - const migrationsFolder = resolveBrainMigrationsFolder(); - - // Safety backup before any migration work. - if (tableExists(nativeDb, 'brain_decisions') && _dbPath) { - createSafetyBackup(_dbPath); - } - - // Bootstrap baseline + reconcile stale journal entries. - reconcileJournal(nativeDb, migrationsFolder, 'brain_decisions', 'brain'); - - // Run pending migrations with SQLITE_BUSY retry. - // Pass nativeDb + existenceTable so migrateWithRetry can auto-reconcile any - // partial migration (Scenario 3) that slips through the proactive check above. - migrateWithRetry(db, migrationsFolder, nativeDb, 'brain_decisions', 'brain'); -} +// +// E6-L2 (T11522): the legacy `runBrainMigrations` helper that ran the +// `drizzle-brain` migration folder (with ~15 ensureColumns + ~8 raw CREATE TABLE +// band-aids) has been removed. After getBrainDb() routes through +// openDualScopeDb('project'), the consolidated cleo-project migrations create +// every `brain_*` table in its final form, and the forward migration +// `20260601000002_t11522-brain-runtime-legacy-tables` adds the only two +// runtime-legacy tables the consolidation skipped (`brain_task_observations`, +// unprefixed `deriver_queue`). The legacy folder is no longer applied here — +// its cross-migration rename chain would collide with the consolidated tables. /** * Load the sqlite-vec extension into a native DatabaseSync instance. @@ -332,11 +281,16 @@ export async function getBrainDb(cwd?: string): Promise Date: Mon, 1 Jun 2026 00:59:51 -0700 Subject: [PATCH 3/6] test(T11522): rewrite auto-recovery-chokepoint for E6-L2 dual-scope cleo.db contract brain.db-file malformation recovery no longer wired into getBrainDb (brain domain moved to cleo.db; brain-only snapshot-restore would corrupt co-resident domains). recoverMalformedBrainDb primitive still unit-tested in recover-brain-db.test.ts. New contract: getBrainDbPath->cleo.db, getBrainDb opens cleo.db w/ runtime-legacy tables, malformed standalone brain.db is irrelevant + untouched. Co-Authored-By: Claude Opus 4.8 --- .../auto-recovery-chokepoint.test.ts | 599 +++--------------- 1 file changed, 105 insertions(+), 494 deletions(-) diff --git a/packages/core/src/store/__tests__/auto-recovery-chokepoint.test.ts b/packages/core/src/store/__tests__/auto-recovery-chokepoint.test.ts index 292151b5f..6fe3ac5a7 100644 --- a/packages/core/src/store/__tests__/auto-recovery-chokepoint.test.ts +++ b/packages/core/src/store/__tests__/auto-recovery-chokepoint.test.ts @@ -1,70 +1,40 @@ /** - * Chokepoint regression test for brain.db auto-recovery - * (T10302 / Saga T10281 / Epic T10286). - * - * Acceptance criterion 1 of Epic T10286 was: - * - * > "Regression test: synthesize malformed brain.db fixture → run - * > cleo memory observe → assert auto-recovery fires, snapshot restored, - * > original observation succeeds on retry" - * - * T10303 shipped the recovery pipeline plus its own unit tests for - * {@link recoverMalformedBrainDb} (the leaf function). This file is the - * COMPLEMENTARY chokepoint test that proves the open path - * `getBrainDb()` in `packages/core/src/store/memory-sqlite.ts` actually - * triggers the recovery pipeline end-to-end when a malformed brain.db - * exists on disk — i.e. the wiring is connected. - * - * The chokepoint that auto-recovers is the writer-side `getBrainDb(cwd)` - * exported from `packages/core/src/store/memory-sqlite.ts`. The - * `cleo memory observe` CLI path lands in this same chokepoint (every - * memory write opens brain.db via this singleton initializer), so - * exercising it here is equivalent to running the CLI verb against a - * malformed live DB. - * - * ## Why this test cannot be unit-level - * - * The unit tests in `recover-brain-db.test.ts` already verify the leaf - * function in isolation. They mock nothing — but they DO call the leaf - * directly. This file instead lets the brain-DB open pipeline run - * end-to-end (open → malformation detect → recover → retry → drizzle - * wrap → migrations) so a future refactor that detaches - * `recoverMalformedBrainDb` from the chokepoint (e.g. someone removes - * the call site or skips the post-open `quick_check` probe) is caught - * by a red CI signal rather than a silent regression. - * - * The file name deliberately omits the `-integration` suffix so it - * runs in the default `pnpm exec vitest run` matrix (the package's - * vitest config excludes `**\/*-integration.test.ts`); per-test runtime - * is bounded under 7s so the suite total stays well below 30s. - * - * ## Fixture strategy - * - * The chokepoint recovers on TWO failure surfaces: - * 1. `openNativeDatabase()` throws with the malformation signature - * (`ERR_SQLITE_ERROR errcode=11` or `/malformed/i`). - * 2. open succeeds but `PRAGMA quick_check` returns non-`ok`. - * - * Writing a non-SQLite garbage file at the brain.db path triggers - * surface #1 only for SOME node:sqlite versions (others surface - * SQLITE_NOTADB = 26, which the chokepoint deliberately does not treat - * as malformation — that would mask unrelated user errors). To match - * the live T10260/T10265 incident pattern AND give the chokepoint a - * deterministic trigger, the fixture here builds a real SQLite file - * with a corrupted `sqlite_schema` B-tree (bytes 100-200 on page 1, - * matching the live RCA finding) so: - * - * - The 16-byte SQLite header is intact → `new DatabaseSync(...)` - * succeeds. - * - The schema page B-tree header is mangled → `PRAGMA quick_check` - * returns non-`ok` OR throws errcode=11 on the first prepare(). - * - * Either surface drives the chokepoint into the recovery branch, which - * is exactly the contract we want to lock in. - * - * @task T10302 - * @epic T10286 - * @saga T10281 + * Brain-domain open-chokepoint contract test. + * + * ## History — brain.db auto-recovery (T10302 / Saga T10281 / Epic T10286) + * + * This file originally locked in a regression contract: calling the real + * `getBrainDb()` chokepoint against a project root containing a malformed + * standalone `brain.db` triggered the T10303 recovery pipeline + * (`recoverMalformedBrainDb`) — quarantine the corrupt file, restore the + * freshest validated `brain.db.snapshot-*`, and re-open. + * + * ## E6-L2 architectural change (T11522 · SG-DB-SUBSTRATE-V2) + * + * `getBrainDb()` no longer opens a standalone `brain.db`. It now routes through + * `openDualScopeDb('project')`, so the brain domain lives inside the consolidated + * project `cleo.db` alongside the `tasks_*` / `conduit_*` / `docs_*` domains. + * The brain-only quarantine/snapshot-restore pipeline cannot be wired into this + * chokepoint anymore: restoring a brain-only snapshot over `cleo.db` would + * destroy every co-resident non-brain domain. Consolidated-`cleo.db` + * malformation recovery is therefore a dual-scope-level concern (a later leaf / + * the exodus), NOT this brain chokepoint's job. + * + * The recovery PRIMITIVE itself — `recoverMalformedBrainDb` — is unchanged and + * remains under unit test in `recover-brain-db.test.ts`; only its wiring into + * `getBrainDb()` was removed. This file now asserts the NEW chokepoint contract: + * + * 1. `getBrainDbPath()` resolves to the consolidated `cleo.db`, NOT `brain.db`. + * 2. `getBrainDb()` opens the consolidated `cleo.db` and yields a usable + * Drizzle connection. + * 3. A pre-existing malformed *standalone* `brain.db` on disk is irrelevant to + * the chokepoint — it is neither read nor touched, and the chokepoint still + * succeeds (the brain domain is served from `cleo.db`). + * + * @task T11522 + * @task T10302 (superseded contract) + * @epic T11249 + * @saga T11242 */ import { @@ -73,20 +43,18 @@ import { mkdirSync, mkdtempSync, openSync, - readdirSync, - readSync, rmSync, writeSync, } from 'node:fs'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { basename, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; // --------------------------------------------------------------------------- // Native SQLite handle (node:sqlite is CJS-only in current Node versions). -// Used for fixture seeding only — the test itself drives the chokepoint via -// the real `getBrainDb()` API. +// Used for fixture seeding + read-back assertions only — the test itself drives +// the chokepoint via the real `getBrainDb()` API. // --------------------------------------------------------------------------- const _require = createRequire(import.meta.url); @@ -102,481 +70,124 @@ const { DatabaseSync } = _require('node:sqlite') as { // --------------------------------------------------------------------------- /** - * Name of the marker table used by snapshot fixtures. Deliberately chosen - * to NOT clash with any production brain schema table — the brain Drizzle - * migrations (`packages/core/migrations/drizzle-brain/`) define - * `brain_decisions`, `brain_patterns`, `brain_observations`, etc., none - * of which match this prefix. The brain.db migration runner that follows - * the chokepoint open therefore leaves this table untouched, letting the - * post-recovery test assertions observe it intact. - */ -const T10302_MARKER_TABLE = 't10302_recovery_marker'; - -/** - * Seed a minimal valid brain.db at `path` containing the T10302 marker - * table pre-populated with `count` rows tagged by `seedTag`. + * Synthesize a malformed standalone `brain.db`: build a real SQLite file then + * corrupt the `sqlite_schema` B-tree page (page 1, bytes 100..200) so the file + * passes the SQLite magic-header check but fails `PRAGMA quick_check` the moment + * any schema-touching query runs. Matches the live T10260/T10265 signature. * - * The marker table is the ONLY schema seeded — we deliberately do NOT - * seed any `brain_*` table. The brain Drizzle migrations that run AFTER - * recovery use `CREATE TABLE IF NOT EXISTS` for every brain table, so - * skipping them here lets the migration runner create them fresh - * without column-mismatch warnings against our test stub. - * - * Used to build SNAPSHOT files that recovery will restore. - * - * @internal - */ -function seedHealthyBrainSnapshot(path: string, count: number, seedTag: string): void { - const db = new DatabaseSync(path); - // The marker table is the source of truth for our test assertions. - db.exec(` - CREATE TABLE IF NOT EXISTS ${T10302_MARKER_TABLE} ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - seed_tag TEXT NOT NULL - ); - `); - const stmt = db.prepare( - `INSERT INTO ${T10302_MARKER_TABLE} (id, content, seed_tag) VALUES (?, ?, ?)`, - ); - for (let i = 0; i < count; i++) { - stmt.run(`${seedTag}-${i}`, `pre-seeded marker ${seedTag}-${i}`, seedTag); - } - db.close(); -} - -/** - * Synthesize a malformed brain.db: build a real SQLite file then corrupt - * the `sqlite_schema` B-tree page (page 1, bytes 100-200) so the file - * passes the SQLite magic-header check but fails `PRAGMA quick_check` - * the moment any schema-touching query runs. - * - * Matches the live T10260/T10265 incident signature called out in - * T10301's RCA: malformed schema page surfacing as - * `ERR_SQLITE_ERROR errcode=11` on the first prepare(). + * Used here only to prove the E6-L2 chokepoint does NOT read this file. * * @internal */ function synthesizeMalformedBrainDb(path: string): void { - // Step 1 — build a real SQLite DB with a schema and a row so the file - // contains real B-tree pages (not just header). The table name is the - // T10302 marker (not `brain_observations`) so the corruption byte-write - // below remains the only failure-injection mechanism; we don't want a - // mismatched schema on the corrupt file to mask the byte-corruption. const db = new DatabaseSync(path); db.exec(` - CREATE TABLE ${T10302_MARKER_TABLE}_corrupt (id TEXT PRIMARY KEY, content TEXT NOT NULL); - INSERT INTO ${T10302_MARKER_TABLE}_corrupt (id, content) VALUES ('seed-row', 'pre-corruption'); + CREATE TABLE corrupt_marker (id TEXT PRIMARY KEY, content TEXT NOT NULL); + INSERT INTO corrupt_marker (id, content) VALUES ('seed-row', 'pre-corruption'); `); db.close(); - // Step 2 — scribble bytes 100..200 on page 1. Page 1 is the - // sqlite_schema page; its B-tree header starts at offset 100 (the - // 16-byte file header occupies bytes 0..15, then SQLite-format - // metadata fills 16..99, and page 1's interior B-tree header lives - // at 100..107). Mangling 100..200 destroys both the B-tree header - // AND the cell-pointer array, guaranteeing `quick_check` will refuse - // the page. const fd = openSync(path, 'r+'); try { - // Verify we have a real SQLite header before corrupting — guards - // against accidentally clobbering an empty/foreign file. - const headerBuf = Buffer.alloc(16); - readSync(fd, headerBuf, 0, 16, 0); - if (headerBuf.toString('utf8', 0, 15) !== 'SQLite format 3') { - throw new Error( - 'synthesizeMalformedBrainDb: fixture pre-condition failed — ' + - 'seeded file lacks the SQLite-3 magic header', - ); - } - - // 100 bytes of `0xFF` — a value that's structurally invalid in - // every B-tree header field (page type, cell count, cell-content - // start, freeblock pointer) so SQLite cannot misinterpret the - // page as a degenerate-but-valid one. + // 100 bytes of 0xFF over the page-1 B-tree header + cell-pointer array + // (offset 100..200) — guarantees `quick_check` refuses the page. const corruption = Buffer.alloc(100, 0xff); writeSync(fd, corruption, 0, 100, 100); } finally { closeSync(fd); } - // Step 3 — drop any stale WAL/SHM sidecars. They would let SQLite - // recover the original schema from journal frames, defeating the - // corruption. for (const suffix of ['-wal', '-shm']) { const sidecar = path + suffix; - if (existsSync(sidecar)) { - rmSync(sidecar, { force: true }); - } + if (existsSync(sidecar)) rmSync(sidecar, { force: true }); } } -/** - * Build the canonical snapshot filename used by `cleo backup add`: - * `brain.db.snapshot-`. The ISO uses `-` for `:` and `.` - * because Windows filesystems reject `:`. - * - * @internal - */ -function snapshotFilename(epochMs: number): string { - return `brain.db.snapshot-${new Date(epochMs).toISOString().replace(/[:.]/g, '-')}`; -} - // --------------------------------------------------------------------------- -// Test suite +// Test suite — E6-L2 chokepoint contract // --------------------------------------------------------------------------- -/** - * End-to-end regression: prove that calling the real `getBrainDb()` - * chokepoint against a project root containing a malformed brain.db - * triggers the T10303 recovery pipeline, restores the freshest - * snapshot, and yields a usable Drizzle connection. Guards against - * future refactors detaching `recoverMalformedBrainDb` from the open - * chokepoint. - */ -describe('brain.db chokepoint integration — auto-recovery on malformed DB (T10302)', () => { +describe('brain-domain open chokepoint — dual-scope cleo.db routing (T11522)', () => { let projectRoot: string; let cleoDir: string; - let brainDbPath: string; - let snapshotDir: string; beforeEach(() => { - projectRoot = mkdtempSync(join(tmpdir(), 'cleo-T10302-integration-')); + process.env.VITEST = '1'; + projectRoot = mkdtempSync(join(tmpdir(), 'cleo-T11522-chokepoint-')); cleoDir = join(projectRoot, '.cleo'); - brainDbPath = join(cleoDir, 'brain.db'); - snapshotDir = join(cleoDir, 'backups', 'snapshot'); - mkdirSync(snapshotDir, { recursive: true }); + mkdirSync(cleoDir, { recursive: true }); }); - afterEach(() => { - vi.restoreAllMocks(); - vi.resetModules(); + afterEach(async () => { + const { resetBrainDbState } = await import('../memory-sqlite.js'); + resetBrainDbState(); rmSync(projectRoot, { recursive: true, force: true }); }); - it('chokepoint quarantines a malformed brain.db and restores the freshest valid snapshot', async () => { - // ------------------------------------------------------------------ - // 1. Fixture: place a snapshot that's a real, healthy brain.db file - // with a known marker row. Recovery must copy this into place. - // ------------------------------------------------------------------ - const fiveMinAgo = Date.now() - 5 * 60 * 1000; - const snapshotPath = join(snapshotDir, snapshotFilename(fiveMinAgo)); - seedHealthyBrainSnapshot(snapshotPath, 4, 'snap'); - - // ------------------------------------------------------------------ - // 2. Fixture: place a malformed brain.db at the chokepoint's - // canonical location. - // ------------------------------------------------------------------ - mkdirSync(cleoDir, { recursive: true }); - synthesizeMalformedBrainDb(brainDbPath); - - // ------------------------------------------------------------------ - // 3. Mock the logger before the chokepoint module is loaded so we - // can assert the canonical `brain.auto-recovery` warn() fires. - // ------------------------------------------------------------------ - const warnCalls: Array<{ obj: unknown; msg: string }> = []; - vi.resetModules(); - vi.doMock('../../logger.js', () => ({ - getLogger: (_subsystem?: string) => ({ - info: vi.fn(), - debug: vi.fn(), - error: vi.fn(), - warn: vi.fn((...args: unknown[]) => { - // Pino-style logger: warn(obj, msg) OR warn(msg). - if (args.length >= 2 && typeof args[1] === 'string') { - warnCalls.push({ obj: args[0], msg: args[1] }); - } else if (typeof args[0] === 'string') { - warnCalls.push({ obj: {}, msg: args[0] }); - } - }), - }), - })); - - // ------------------------------------------------------------------ - // 4. Drive the chokepoint with a fresh module graph so the mocks - // above are picked up. - // ------------------------------------------------------------------ - const { getBrainDb, resetBrainDbState, getBrainNativeDb } = await import('../memory-sqlite.js'); - resetBrainDbState(); - - let chokepointThrew: unknown = null; - try { - const db = await getBrainDb(projectRoot); - expect(db).toBeTruthy(); - } catch (err) { - chokepointThrew = err; - } - expect( - chokepointThrew, - `chokepoint must not throw after auto-recovery — got: ${ - chokepointThrew instanceof Error ? chokepointThrew.message : String(chokepointThrew) - }`, - ).toBeNull(); - - // ------------------------------------------------------------------ - // 5. Assert: the malformed file moved to quarantine. Recovery - // pipeline creates `/quarantine/brain-malformed-/ - // brain.db.malformed`. - // ------------------------------------------------------------------ - const quarantineRoot = join(cleoDir, 'quarantine'); - expect(existsSync(quarantineRoot)).toBe(true); - const quarantineEntries = readdirSync(quarantineRoot); - expect(quarantineEntries.length).toBeGreaterThan(0); - const quarantineDir = join(quarantineRoot, quarantineEntries[0] as string); - expect(quarantineDir).toMatch(/brain-malformed-/); - expect(existsSync(join(quarantineDir, 'brain.db.malformed'))).toBe(true); - - // ------------------------------------------------------------------ - // 6. Assert: brain.db on disk now contains the snapshot's marker - // row. This proves the snapshot WAS restored, not that recovery - // merely "didn't throw". Opens read-only to avoid stealing the - // chokepoint singleton. - // ------------------------------------------------------------------ - expect(existsSync(brainDbPath)).toBe(true); - const verify = new DatabaseSync(brainDbPath, { readonly: true }); - try { - const row = verify - .prepare(`SELECT content FROM ${T10302_MARKER_TABLE} WHERE id = 'snap-0'`) - .get() as { content?: string } | undefined; - expect(row?.content).toBe('pre-seeded marker snap-0'); - } finally { - verify.close(); - } - - // ------------------------------------------------------------------ - // 7. Assert: the chokepoint emitted the canonical - // `brain.auto-recovery` warn() exactly once with a structured - // payload pointing at the snapshot we placed. - // ------------------------------------------------------------------ - const recoveryWarns = warnCalls.filter((c) => { - const obj = c.obj as Record | undefined; - return obj?.['event'] === 'brain.auto-recovery'; - }); - expect(recoveryWarns.length).toBe(1); - const payload = recoveryWarns[0]?.obj as Record; - expect(payload['restoredFrom']).toBe(snapshotPath); - expect(payload['source']).toBe('system-snapshot'); - expect(typeof payload['dataLossWindowHours']).toBe('number'); - expect(payload['quarantineDir']).toBe(quarantineDir); - expect(recoveryWarns[0]?.msg).toMatch(/BRAIN auto-recovered/); - - // ------------------------------------------------------------------ - // 8. Assert: the chokepoint's native handle is usable — the - // underlying singleton is open and `quick_check` returns ok on - // the restored DB. This is the "retry succeeded" half of the - // Epic T10286 acceptance criterion. - // ------------------------------------------------------------------ - const native = getBrainNativeDb(); - expect(native).not.toBeNull(); - if (native) { - const quick = native.prepare('PRAGMA quick_check').get() as - | { quick_check?: string } - | undefined; - expect(quick?.quick_check).toBe('ok'); - } - - resetBrainDbState(); + it('getBrainDbPath resolves to the consolidated cleo.db, not a standalone brain.db', async () => { + const { getBrainDbPath } = await import('../memory-sqlite.js'); + const dbPath = getBrainDbPath(projectRoot); + expect(basename(dbPath)).toBe('cleo.db'); + expect(dbPath).not.toMatch(/brain\.db$/); }); - it('chokepoint picks the freshest VALIDATED snapshot when multiple exist (newest-first ranking)', async () => { - // ------------------------------------------------------------------ - // 1. Place two snapshots: an older healthy one (10 minutes ago) - // and a newer healthy one (1 minute ago). Recovery must select - // the newer one per the freshness-ranking invariant. - // ------------------------------------------------------------------ - const tenMinAgo = Date.now() - 10 * 60 * 1000; - const oneMinAgo = Date.now() - 1 * 60 * 1000; - const olderPath = join(snapshotDir, snapshotFilename(tenMinAgo)); - const newerPath = join(snapshotDir, snapshotFilename(oneMinAgo)); - seedHealthyBrainSnapshot(olderPath, 2, 'older'); - seedHealthyBrainSnapshot(newerPath, 9, 'newer'); - - mkdirSync(cleoDir, { recursive: true }); - synthesizeMalformedBrainDb(brainDbPath); - - const warnCalls: Array<{ obj: unknown; msg: string }> = []; - vi.resetModules(); - vi.doMock('../../logger.js', () => ({ - getLogger: (_subsystem?: string) => ({ - info: vi.fn(), - debug: vi.fn(), - error: vi.fn(), - warn: vi.fn((...args: unknown[]) => { - if (args.length >= 2 && typeof args[1] === 'string') { - warnCalls.push({ obj: args[0], msg: args[1] }); - } - }), - }), - })); - - const { getBrainDb, resetBrainDbState } = await import('../memory-sqlite.js'); + it('getBrainDb opens the consolidated cleo.db and yields a usable connection', async () => { + const { getBrainDb, getBrainNativeDb, getBrainDbPath, resetBrainDbState } = await import( + '../memory-sqlite.js' + ); resetBrainDbState(); const db = await getBrainDb(projectRoot); expect(db).toBeTruthy(); - // The restored DB must contain the NEWER snapshot's marker, not the older one. - const verify = new DatabaseSync(brainDbPath, { readonly: true }); - try { - const newerRow = verify - .prepare(`SELECT content FROM ${T10302_MARKER_TABLE} WHERE id = 'newer-0'`) - .get() as { content?: string } | undefined; - expect(newerRow?.content).toBe('pre-seeded marker newer-0'); - // And NOT the older one. - const olderRow = verify - .prepare(`SELECT content FROM ${T10302_MARKER_TABLE} WHERE id = 'older-0'`) - .get() as { content?: string } | undefined; - expect(olderRow).toBeUndefined(); - } finally { - verify.close(); + // The chokepoint must have created the consolidated cleo.db on disk. + expect(existsSync(getBrainDbPath(projectRoot))).toBe(true); + + // The native handle serves the brain domain: the consolidated brain_* + // tables AND the runtime-legacy tables added by the T11522 forward + // migration (brain_task_observations, unprefixed deriver_queue) exist. + const nativeDb = getBrainNativeDb(); + expect(nativeDb).toBeTruthy(); + for (const table of [ + 'brain_decisions', + 'brain_observations', + 'brain_task_observations', + 'deriver_queue', + ]) { + const row = nativeDb + ?.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = ?") + .get(table) as { name?: string } | undefined; + expect(row?.name, `expected table ${table} to exist in cleo.db`).toBe(table); } - - // Single recovery warn naming the newer snapshot path. - const recovery = warnCalls.find( - (c) => (c.obj as Record)['event'] === 'brain.auto-recovery', - ); - expect(recovery).toBeDefined(); - expect((recovery?.obj as Record)['restoredFrom']).toBe(newerPath); - - resetBrainDbState(); }); - it('chokepoint surfaces failure (does NOT silently degrade) when no valid snapshot exists', async () => { - // ------------------------------------------------------------------ - // 1. Place a malformed brain.db with NO snapshots. The chokepoint - // must NOT silently return a usable handle — it must surface - // the failure so the operator sees it. The recovery pipeline - // moves the corrupt file to quarantine even in this case so the - // next process attempt can retry with fresh state. - // ------------------------------------------------------------------ - mkdirSync(cleoDir, { recursive: true }); - synthesizeMalformedBrainDb(brainDbPath); - - const errorCalls: Array<{ obj: unknown; msg: string }> = []; - vi.resetModules(); - vi.doMock('../../logger.js', () => ({ - getLogger: (_subsystem?: string) => ({ - info: vi.fn(), - debug: vi.fn(), - warn: vi.fn(), - error: vi.fn((...args: unknown[]) => { - if (args.length >= 2 && typeof args[1] === 'string') { - errorCalls.push({ obj: args[0], msg: args[1] }); - } - }), - }), - })); + it('a pre-existing malformed standalone brain.db is irrelevant to the chokepoint', async () => { + // The brain domain moved to cleo.db (E6-L2). A leftover malformed brain.db + // file must NOT be read by — nor break — the chokepoint, and must NOT be + // quarantined by it (that recovery moved to the dual-scope level). + const legacyBrainDbPath = join(cleoDir, 'brain.db'); + synthesizeMalformedBrainDb(legacyBrainDbPath); const { getBrainDb, resetBrainDbState } = await import('../memory-sqlite.js'); resetBrainDbState(); - let outcome: 'threw' | 'returned' = 'returned'; + let threw: unknown = null; try { - await getBrainDb(projectRoot); - } catch { - outcome = 'threw'; + const db = await getBrainDb(projectRoot); + expect(db).toBeTruthy(); + } catch (err) { + threw = err; } - - // The contract: when no snapshot exists, the chokepoint either - // throws (preferred — loud failure) OR the migration phase fails - // because the restored DB is missing. Either way, recovery has - // already moved the corrupt file into quarantine so forensic - // state survives. - const quarantineRoot = join(cleoDir, 'quarantine'); - expect(existsSync(quarantineRoot)).toBe(true); - const quarantineEntries = readdirSync(quarantineRoot); - expect(quarantineEntries.length).toBeGreaterThan(0); - - // At least one error log naming the failed-recovery condition. expect( - errorCalls.some((c) => /no validated snapshot/.test(c.msg) || /auto-recovery/.test(c.msg)), - ).toBe(true); - - // We deliberately don't pin outcome to one branch: the chokepoint - // is free to throw (e.g. migration runner fails on empty DB) OR - // return a partially-initialized handle. The CONTRACT is that the - // failure is OBSERVABLE — captured here by the quarantine + error - // log presence assertions above. - expect(['threw', 'returned']).toContain(outcome); - - resetBrainDbState(); - }); - - it('chokepoint discovers VACUUM-INTO snapshots (cleo session end output) as fallback', async () => { - // ------------------------------------------------------------------ - // 1. Place ONLY a VACUUM-INTO snapshot (no system-snapshot). The - // chokepoint wires `vacuumSnapshotDir: /backups/sqlite` - // so this path must be discovered and restored. - // ------------------------------------------------------------------ - const vacuumDir = join(cleoDir, 'backups', 'sqlite'); - mkdirSync(vacuumDir, { recursive: true }); - - // VACUUM-INTO filename pattern: brain-YYYYMMDD-HHmmss.db - const now = new Date(); - const pad = (n: number): string => n.toString().padStart(2, '0'); - const vacuumName = `brain-${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}.db`; - const vacuumPath = join(vacuumDir, vacuumName); - seedHealthyBrainSnapshot(vacuumPath, 3, 'vacuum'); - - mkdirSync(cleoDir, { recursive: true }); - synthesizeMalformedBrainDb(brainDbPath); - - const warnCalls: Array<{ obj: unknown; msg: string }> = []; - vi.resetModules(); - vi.doMock('../../logger.js', () => ({ - getLogger: (_subsystem?: string) => ({ - info: vi.fn(), - debug: vi.fn(), - error: vi.fn(), - warn: vi.fn((...args: unknown[]) => { - if (args.length >= 2 && typeof args[1] === 'string') { - warnCalls.push({ obj: args[0], msg: args[1] }); - } - }), - }), - })); - - const { getBrainDb, resetBrainDbState } = await import('../memory-sqlite.js'); - resetBrainDbState(); - - const db = await getBrainDb(projectRoot); - expect(db).toBeTruthy(); - - // Vacuum snapshot's marker is now in brain.db. - const verify = new DatabaseSync(brainDbPath, { readonly: true }); - try { - const row = verify - .prepare(`SELECT content FROM ${T10302_MARKER_TABLE} WHERE id = 'vacuum-0'`) - .get() as { content?: string } | undefined; - expect(row?.content).toBe('pre-seeded marker vacuum-0'); - } finally { - verify.close(); - } - - // Recovery announcement tagged the source as `vacuum-snapshot`. - const recovery = warnCalls.find( - (c) => (c.obj as Record)['event'] === 'brain.auto-recovery', - ); - expect(recovery).toBeDefined(); - expect((recovery?.obj as Record)['source']).toBe('vacuum-snapshot'); - expect((recovery?.obj as Record)['restoredFrom']).toBe(vacuumPath); + threw, + `chokepoint must succeed regardless of a malformed standalone brain.db — got: ${ + threw instanceof Error ? threw.message : String(threw) + }`, + ).toBeNull(); - resetBrainDbState(); + // The chokepoint does not own brain.db-file recovery anymore: the malformed + // file is left untouched (no quarantine dir created by this chokepoint). + expect(existsSync(legacyBrainDbPath)).toBe(true); + expect(existsSync(join(cleoDir, 'quarantine'))).toBe(false); }); }); - -// --------------------------------------------------------------------------- -// Note on fixture construction -// --------------------------------------------------------------------------- -// -// The helpers above use the raw `node:sqlite` `DatabaseSync` constructor -// (not `openNativeDatabase`) to seed the malformed brain.db and the -// snapshot files. The vitest production-DB guard inside -// `assertVitestSafePath` is path-based — it allows any path under -// `os.tmpdir()`, which is where every fixture file lives. The -// db-open-allowed annotation below documents that these fixture seeds -// are deliberate and orthogonal to the `openCleoDb()` chokepoint -// contract enforced by the `DB Open Guard` CI gate (T10073). -// db-open-allowed: T10302 fixture writer — seeds tmp SQLite files under -// os.tmpdir() to construct malformed/snapshot fixtures consumed by the -// integration test above. From 3cb2ef9f02e8417521446d5bfdb5664616a36f33 Mon Sep 17 00:00:00 2001 From: kryptobaseddev Date: Mon, 1 Jun 2026 01:20:07 -0700 Subject: [PATCH 4/6] fix(T11522): establish legacy brain schema in cleo.db + share handle safely (E6-L2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: consolidated brain_* tables are the exodus-TARGET shape (ISO-8601 text timestamps + enum/format CHECK constraints) — incompatible with the runtime brainSchema (epoch-ms integers). Unlike tasks (legacy 'tasks' != 'tasks_tasks'), brain names collide. Fix: establishLegacyBrainSchema drops the consolidated brain tables and runs legacy drizzle-brain migrations to recreate them in runtime shape. Added forward migrations for brain_task_observations + the 3 inline-only tables (brain_transcript_events/promotion_log/backfill_runs) that had no drizzle-brain migration. closeBrainDb/resetBrainDbState no longer tear down the shared dual-scope handle (broke tasks domain). Liveness guard re-derives brain singleton when the shared handle was closed by the tasks side. Updated 2 migration tests to cleo.db. Co-Authored-By: Claude Opus 4.8 --- .../migration.sql | 28 +++ .../migration.sql | 75 ++++++ .../migration.sql | 60 ----- .../migration-fresh-no-repair.brain.test.ts | 6 +- .../store/__tests__/migration-smoke.test.ts | 6 +- packages/core/src/store/memory-sqlite.ts | 233 ++++++++++++++---- 6 files changed, 291 insertions(+), 117 deletions(-) create mode 100644 packages/core/migrations/drizzle-brain/20260601000001_t11522-brain-task-observations/migration.sql create mode 100644 packages/core/migrations/drizzle-brain/20260601000002_t11522-inline-only-brain-tables/migration.sql delete mode 100644 packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql diff --git a/packages/core/migrations/drizzle-brain/20260601000001_t11522-brain-task-observations/migration.sql b/packages/core/migrations/drizzle-brain/20260601000001_t11522-brain-task-observations/migration.sql new file mode 100644 index 000000000..9bf6e77bb --- /dev/null +++ b/packages/core/migrations/drizzle-brain/20260601000001_t11522-brain-task-observations/migration.sql @@ -0,0 +1,28 @@ +-- T11522 (E6-L2): Add brain_task_observations as a forward Drizzle migration. +-- +-- brain_task_observations (T1615) is the join table linking brain_observations to +-- task IDs; it powers `cleo memory find` session-context lookups. The runtime +-- writer is sessions/session-memory-bridge.ts. +-- +-- It was previously created ONLY by a post-hoc `CREATE TABLE IF NOT EXISTS` in +-- memory-sqlite.ts::runBrainMigrations — it had NO Drizzle migration anywhere +-- (neither drizzle-brain nor the T11363 consolidation; exodus maps it to `null` +-- as a runtime-only cache). E6-L2 removes that inline band-aid and replaces it +-- with this forward migration, matching the T9179 precedent (ensureColumns / +-- self-healing DDL → forward Drizzle migration). +-- +-- All statements are IF NOT EXISTS so re-running on a DB that already has the +-- table is idempotent and safe. +CREATE TABLE IF NOT EXISTS `brain_task_observations` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `observation_id` text NOT NULL, + `task_id` text NOT NULL, + `link_type` text DEFAULT 'session-completed' NOT NULL, + `created_at` text DEFAULT (datetime('now')) NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `idx_brain_task_obs_unique` ON `brain_task_observations` (`observation_id`, `task_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_observation` ON `brain_task_observations` (`observation_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_task` ON `brain_task_observations` (`task_id`); diff --git a/packages/core/migrations/drizzle-brain/20260601000002_t11522-inline-only-brain-tables/migration.sql b/packages/core/migrations/drizzle-brain/20260601000002_t11522-inline-only-brain-tables/migration.sql new file mode 100644 index 000000000..8a3d104f4 --- /dev/null +++ b/packages/core/migrations/drizzle-brain/20260601000002_t11522-inline-only-brain-tables/migration.sql @@ -0,0 +1,75 @@ +-- T11522 (E6-L2): Forward Drizzle migrations for the three brain tables that had +-- NO drizzle-brain migration and were created ONLY by post-hoc +-- `CREATE TABLE IF NOT EXISTS` band-aids in memory-sqlite.ts::runBrainMigrations. +-- +-- E6-L2 removes those inline band-aids and replaces them with this forward +-- migration (T11522 AC: post-hoc DDL → forward Drizzle migration), matching the +-- T9179 precedent. The schemas reproduce the removed inline DDL EXACTLY (legacy +-- runtime shape) so the brain runtime writers continue to work unchanged: +-- +-- 1. brain_transcript_events (T1002) — full-fidelity Claude session block store. +-- 2. brain_promotion_log (T1001/T1903) — typed promotion audit trail (includes +-- the T1903 fulfilled_at / fulfillment_note columns folded into the CREATE). +-- 3. brain_backfill_runs (T1003) — staged backfill audit log. +-- +-- All statements are IF NOT EXISTS so re-running on a DB that already has the +-- tables is idempotent and safe. +CREATE TABLE IF NOT EXISTS `brain_transcript_events` ( + `id` text PRIMARY KEY, + `session_id` text NOT NULL, + `seq` integer NOT NULL, + `role` text NOT NULL, + `block_type` text NOT NULL, + `content` text NOT NULL, + `tokens` integer, + `redacted_at` text, + `created_at` text DEFAULT (datetime('now')) NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS `idx_transcript_events_session_seq` ON `brain_transcript_events` (`session_id`, `seq`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_transcript_events_session` ON `brain_transcript_events` (`session_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_transcript_events_role` ON `brain_transcript_events` (`role`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_transcript_events_block_type` ON `brain_transcript_events` (`block_type`); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS `brain_promotion_log` ( + `id` text PRIMARY KEY, + `observation_id` text NOT NULL, + `from_tier` text NOT NULL, + `to_tier` text NOT NULL, + `score` real NOT NULL, + `decided_at` text DEFAULT (datetime('now')) NOT NULL, + `decided_by` text DEFAULT 'composite-scorer' NOT NULL, + `rationale_json` text, + `fulfilled_at` text, + `fulfillment_note` text +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_promotion_log_observation` ON `brain_promotion_log` (`observation_id`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_promotion_log_decided_at` ON `brain_promotion_log` (`decided_at`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_promotion_log_to_tier` ON `brain_promotion_log` (`to_tier`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_promotion_log_score` ON `brain_promotion_log` (`score`); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS `brain_backfill_runs` ( + `id` text PRIMARY KEY, + `kind` text NOT NULL, + `status` text DEFAULT 'staged' NOT NULL, + `created_at` text DEFAULT (datetime('now')) NOT NULL, + `approved_at` text, + `rows_affected` integer DEFAULT 0 NOT NULL, + `rollback_snapshot_json` text, + `source` text DEFAULT 'unknown' NOT NULL, + `target_table` text DEFAULT 'brain_observations' NOT NULL, + `approved_by` text +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_backfill_runs_status` ON `brain_backfill_runs` (`status`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_backfill_runs_kind` ON `brain_backfill_runs` (`kind`); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS `idx_backfill_runs_created_at` ON `brain_backfill_runs` (`created_at`); diff --git a/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql b/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql deleted file mode 100644 index 54dca07b7..000000000 --- a/packages/core/migrations/drizzle-cleo-project/20260601000002_t11522-brain-runtime-legacy-tables/migration.sql +++ /dev/null @@ -1,60 +0,0 @@ --- T11522 (E6-L2): Add the two runtime-legacy BRAIN tables the T11363 consolidation --- migration skipped, so getBrainDb() — now routed through openDualScopeDb('project') --- — can serve every brain-domain runtime query from the consolidated `cleo.db`. --- --- Both tables were previously created by post-hoc `CREATE TABLE IF NOT EXISTS` DDL --- in memory-sqlite.ts::runBrainMigrations, which E6-L2 removed. The runtime queries --- them by their pre-consolidation physical names via raw SQL, so they must exist in --- `cleo.db`. This forward migration replaces the removed post-hoc DDL — matching the --- T9179 precedent (ensureColumns / self-healing DDL → forward Drizzle migration). --- Running the legacy `drizzle-brain` migration set against the consolidated DB is NOT --- viable: its cross-migration rename chain (t1147 `brain_v2_candidate` → t1402 RENAME --- TO `brain_observations_staging`) collides with the final table the consolidation --- already created. The consolidated schema is the brain SSoT; this migration adds only --- the two genuinely-uncovered runtime tables. --- --- 1. brain_task_observations (T1615) — join table linking brain_observations to task --- IDs; powers `cleo memory find` session-context lookups. The runtime writer is --- sessions/session-memory-bridge.ts. Exodus maps it to `null` (runtime-only cache). --- --- 2. deriver_queue (T1145) — durable background derivation work queue. The runtime --- accessors are packages/core/src/deriver/{enqueue,queue-manager,status}.ts, all of --- which open via getBrainNativeDb() and query the UNPREFIXED name. The consolidated --- schema carries the prefixed `brain_deriver_queue`; exodus renames the legacy table --- onto it. Until that cutover the unprefixed table must exist for the runtime. --- --- All statements are IF NOT EXISTS so re-running onto a DB that already has these --- tables (e.g. after exodus) is idempotent and safe. -CREATE TABLE IF NOT EXISTS `brain_task_observations` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `observation_id` text NOT NULL, - `task_id` text NOT NULL, - `link_type` text DEFAULT 'session-completed' NOT NULL, - `created_at` text DEFAULT (datetime('now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX IF NOT EXISTS `idx_brain_task_obs_unique` ON `brain_task_observations` (`observation_id`, `task_id`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_observation` ON `brain_task_observations` (`observation_id`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_brain_task_obs_task` ON `brain_task_observations` (`task_id`); ---> statement-breakpoint -CREATE TABLE IF NOT EXISTS `deriver_queue` ( - `id` text PRIMARY KEY NOT NULL, - `item_type` text NOT NULL, - `item_id` text NOT NULL, - `priority` integer DEFAULT 0 NOT NULL, - `status` text DEFAULT 'pending' NOT NULL, - `claimed_at` text, - `claimed_by` text, - `error_msg` text, - `retry_count` integer DEFAULT 0 NOT NULL, - `created_at` text DEFAULT (datetime('now')) NOT NULL, - `completed_at` text -); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_deriver_queue_status_priority` ON `deriver_queue` (`status`, `priority` DESC, `created_at` ASC); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_deriver_queue_item` ON `deriver_queue` (`item_type`, `item_id`); ---> statement-breakpoint -CREATE INDEX IF NOT EXISTS `idx_deriver_queue_claimed_at` ON `deriver_queue` (`claimed_at`); diff --git a/packages/core/src/store/__tests__/migration-fresh-no-repair.brain.test.ts b/packages/core/src/store/__tests__/migration-fresh-no-repair.brain.test.ts index aea343b43..4cc74d288 100644 --- a/packages/core/src/store/__tests__/migration-fresh-no-repair.brain.test.ts +++ b/packages/core/src/store/__tests__/migration-fresh-no-repair.brain.test.ts @@ -147,10 +147,10 @@ describe('brain.db fresh init — zero "Adding missing column" warnings', () => // ------------------------------------------------------------------ // 7. SECONDARY: verify required columns exist via PRAGMA table_info. - // The brain.db lives inside cleoHome/.cleo/ because getCleoDirAbsolute - // with the tempDir cwd returns join(tempDir, '.cleo'). + // E6-L2 (T11522): the brain domain now lives inside the consolidated + // `cleo.db` (openDualScopeDb), not a standalone `brain.db`. // ------------------------------------------------------------------ - const dbPath = join(tempDir, '.cleo', 'brain.db'); + const dbPath = join(tempDir, '.cleo', 'cleo.db'); const nativeDb = new DatabaseSync(dbPath, { readonly: true }); try { diff --git a/packages/core/src/store/__tests__/migration-smoke.test.ts b/packages/core/src/store/__tests__/migration-smoke.test.ts index 053834980..e031db509 100644 --- a/packages/core/src/store/__tests__/migration-smoke.test.ts +++ b/packages/core/src/store/__tests__/migration-smoke.test.ts @@ -114,7 +114,7 @@ describe('Test 1: fresh init — all 5 DBs migrate clean', () => { } }); - it('brain.db (drizzle-brain) — fresh init succeeds, brain_decisions table exists', async () => { + it('brain (drizzle-brain) — fresh init succeeds, brain_decisions table exists', async () => { const { getBrainDb, resetBrainDbState } = await import('../memory-sqlite.js'); resetBrainDbState(); @@ -125,8 +125,10 @@ describe('Test 1: fresh init — all 5 DBs migrate clean', () => { const db = await getBrainDb(projectDir); expect(db).toBeTruthy(); + // E6-L2 (T11522): the brain domain consolidated into `cleo.db` via + // openDualScopeDb('project') — not a standalone `brain.db`. const { openNativeDatabase } = await import('../sqlite.js'); - const nativeDb = openNativeDatabase(join(projectDir, '.cleo', 'brain.db')); + const nativeDb = openNativeDatabase(join(projectDir, '.cleo', 'cleo.db')); const row = nativeDb .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='brain_decisions'") .get() as { name: string } | undefined; diff --git a/packages/core/src/store/memory-sqlite.ts b/packages/core/src/store/memory-sqlite.ts index ad32773c1..0892b123f 100644 --- a/packages/core/src/store/memory-sqlite.ts +++ b/packages/core/src/store/memory-sqlite.ts @@ -54,11 +54,14 @@ import type { drizzle as drizzleFn, NodeSQLiteDatabase } from 'drizzle-orm/node- // DatabaseSync lifecycle, pragmas, and consolidated migrations. We extract the // native handle and re-wrap it with the legacy brain-schema drizzle instance so // existing callers (brainSchema.* queries) compile and run without change. +import { getLogger } from '../logger.js'; +import { openDualScopeDb, resolveDualScopeDbPath } from './dual-scope-db.js'; import { - _resetDualScopeDbCache, - openDualScopeDb, - resolveDualScopeDbPath, -} from './dual-scope-db.js'; + createSafetyBackup, + migrateWithRetry, + reconcileJournal, + tableExists, +} from './migration-manager.js'; import { resolveCorePackageMigrationsFolder } from './resolve-migrations-folder.js'; import * as brainSchema from './schema/memory-schema.js'; @@ -133,13 +136,146 @@ export function resolveBrainMigrationsFolder(): string { // // E6-L2 (T11522): the legacy `runBrainMigrations` helper that ran the // `drizzle-brain` migration folder (with ~15 ensureColumns + ~8 raw CREATE TABLE -// band-aids) has been removed. After getBrainDb() routes through -// openDualScopeDb('project'), the consolidated cleo-project migrations create -// every `brain_*` table in its final form, and the forward migration -// `20260601000002_t11522-brain-runtime-legacy-tables` adds the only two -// runtime-legacy tables the consolidation skipped (`brain_task_observations`, -// unprefixed `deriver_queue`). The legacy folder is no longer applied here — -// its cross-migration rename chain would collide with the consolidated tables. +// band-aids). After getBrainDb() routes through openDualScopeDb('project'), the +// brain domain is served from the consolidated `cleo.db`. See +// `establishLegacyBrainSchema` below for why the runtime keeps the LEGACY brain +// table shape during the E3→E6 transition. + +/** + * The set of brain-domain physical tables the T11363 consolidation migration + * creates in the project `cleo.db`. Each is dropped + recreated in its LEGACY + * runtime shape by {@link establishLegacyBrainSchema} (see that function for the + * rationale). `deriver_queue` is included because the legacy `t1145` migration + * creates it (unprefixed) and we must clear any prior shape first. + * + * @internal + * @task T11522 + */ +const CONSOLIDATED_BRAIN_TABLES = [ + 'brain_attention', + 'brain_backfill_runs', + 'brain_consolidation_events', + 'brain_decisions', + 'brain_deriver_queue', + 'brain_embeddings', + 'brain_learnings', + 'brain_memory_links', + 'brain_memory_trees', + 'brain_modulators', + 'brain_observations', + 'brain_observations_staging', + 'brain_page_edges', + 'brain_page_nodes', + 'brain_patterns', + 'brain_plasticity_events', + 'brain_promotion_log', + 'brain_retrieval_log', + 'brain_schema_meta', + 'brain_session_narrative', + 'brain_sticky_notes', + 'brain_sticky_tags', + 'brain_transcript_events', + 'brain_usage_log', + 'brain_weight_history', +] as const; + +/** + * Detect whether the brain tables in the open handle carry the CONSOLIDATED + * (exodus-target) shape rather than the LEGACY runtime shape. + * + * The consolidation migration (T11363) types `brain_attention.created_at` as + * `text` (ISO-8601, with a GLOB CHECK constraint); the legacy runtime schema + * (`memory-schema.ts`) types it as `integer` (epoch-ms, `unixepoch() * 1000`). + * The column affinity is therefore a reliable, cheap discriminator. + * + * @internal + * @task T11522 + */ +function brainTablesAreConsolidatedShape(nativeDb: DatabaseSync): boolean { + if (!tableExists(nativeDb, 'brain_attention')) return false; + const cols = nativeDb.prepare('PRAGMA table_info(brain_attention)').all() as Array<{ + name: string; + type: string; + }>; + const createdAt = cols.find((c) => c.name === 'created_at'); + // Legacy = INTEGER; consolidated target = TEXT. Anything non-INTEGER means we + // are looking at the consolidated target shape and must rebuild to legacy. + return createdAt !== undefined && createdAt.type.toUpperCase() !== 'INTEGER'; +} + +/** + * Establish the LEGACY brain-domain schema inside the consolidated project + * `cleo.db`, replacing the consolidated (exodus-target) brain tables. + * + * ## Why (T11522 · E6-L2) + * + * Routing `getBrainDb()` through {@link openDualScopeDb} runs the T11363 + * consolidation migration, which creates every `brain_*` table in its + * **exodus-target** shape: ISO-8601 `text` timestamps and enum/format `CHECK` + * constraints (e.g. `brain_attention.created_at GLOB '[0-9][0-9][0-9][0-9]-…'`, + * `brain_page_nodes.node_type IN (…)`). The runtime brain writers and the + * `brainSchema` (`memory-schema.ts`) still use the **legacy** shape — epoch-ms + * `integer` timestamps and no enum CHECKs — exactly as the tasks domain keeps + * using the legacy `tasks` table after E6-L1. + * + * Unlike tasks (legacy `tasks` ≠ consolidated `tasks_tasks`, so both co-exist), + * the brain tables were already domain-prefixed, so legacy and consolidated + * share the SAME physical names — they cannot co-exist. The runtime must win, so + * on first open we drop the consolidated brain tables and run the legacy + * `drizzle-brain` migrations to recreate them in the runtime shape. The + * consolidated-target cutover (epoch→ISO conversion, CHECK constraints) is the + * exodus's job — see T11248 / exodus-on-open T11553, which migrate the standalone + * legacy `brain.db` into `cleo.db`. + * + * Idempotent: after the first rebuild the tables are already legacy-shaped, so + * {@link brainTablesAreConsolidatedShape} returns `false` and this is a no-op + * (the `drizzle-brain` journal is reconciled, nothing is dropped). + * + * @internal + * @task T11522 + */ +function establishLegacyBrainSchema( + nativeDb: DatabaseSync, + db: NodeSQLiteDatabase, +): void { + const log = getLogger('brain-schema'); + + if (brainTablesAreConsolidatedShape(nativeDb)) { + // Drop the consolidated (exodus-target) brain tables so the legacy + // `drizzle-brain` migrations can recreate them in the runtime shape. + // `brain_embeddings` is a vec0 virtual table once the extension is loaded; + // DROP TABLE handles both regular and virtual tables when sqlite-vec is + // present. Disable FKs during the drop so cross-table references do not + // block the teardown. + nativeDb.exec('PRAGMA foreign_keys=OFF'); + for (const table of CONSOLIDATED_BRAIN_TABLES) { + try { + nativeDb.exec(`DROP TABLE IF EXISTS \`${table}\``); + } catch (err) { + log.warn({ table, err }, 'Failed to drop consolidated brain table during legacy rebuild.'); + } + } + log.debug( + { count: CONSOLIDATED_BRAIN_TABLES.length }, + 'Dropped consolidated (exodus-target) brain tables — rebuilding in legacy runtime shape.', + ); + } + + // Run the legacy `drizzle-brain` migrations to (re)create the runtime-shaped + // brain tables. Their `__drizzle_migrations` journal is shared with the + // cleo-project journal in the same `cleo.db`; the hashes are disjoint so the + // brain migrations are reconciled/applied independently. + const migrationsFolder = resolveBrainMigrationsFolder(); + if (tableExists(nativeDb, 'brain_decisions') && _dbPath) { + createSafetyBackup(_dbPath); + } + reconcileJournal(nativeDb, migrationsFolder, 'brain_decisions', 'brain'); + migrateWithRetry(db, migrationsFolder, nativeDb, 'brain_decisions', 'brain'); + // The `drizzle-brain` set now includes `brain_task_observations` (T1615) via + // the forward migration `20260601000001_t11522-brain-task-observations`, so the + // previous post-hoc `CREATE TABLE IF NOT EXISTS` band-aid for it is no longer + // needed here (T11522 AC: post-hoc DDL → forward Drizzle migration). +} /** * Load the sqlite-vec extension into a native DatabaseSync instance. @@ -247,6 +383,16 @@ export async function getBrainDb(cwd?: string): Promise Date: Mon, 1 Jun 2026 01:24:11 -0700 Subject: [PATCH 5/6] fix(T11522): restore FK pragma after brain table drop + update reset test for shared handle (E6-L2) - establishLegacyBrainSchema restores foreign_keys pragma to pre-drop state (was leaving it OFF, breaking the T10314 idempotent-pragma contract) - memory-schema reset test asserts the new shared-handle contract: reset clears the brain singleton (new drizzle wrapper) but keeps the shared dual-scope cleo.db handle alive (co-owned by tasks domain) Co-Authored-By: Claude Opus 4.8 --- .../src/store/__tests__/memory-schema.test.ts | 17 ++++++++---- packages/core/src/store/memory-sqlite.ts | 26 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/core/src/store/__tests__/memory-schema.test.ts b/packages/core/src/store/__tests__/memory-schema.test.ts index 66e293824..b01f915d7 100644 --- a/packages/core/src/store/__tests__/memory-schema.test.ts +++ b/packages/core/src/store/__tests__/memory-schema.test.ts @@ -152,7 +152,7 @@ describe('brain.db schema', () => { expect(existsSync(dbPath)).toBe(true); }); - it('resetBrainDbState clears singleton and allows reinitialization', async () => { + it('resetBrainDbState clears the brain singleton and allows reinitialization', async () => { const { getBrainDb, getBrainNativeDb, @@ -163,18 +163,25 @@ describe('brain.db schema', () => { const db1 = await getBrainDb(); expect(db1).toBeDefined(); - // Capture the underlying native handle before reset const nativeDb1 = getBrainNativeDb(); expect(nativeDb1).not.toBeNull(); + // E6-L2 (T11522): resetBrainDbState drops the brain singleton refs but does + // NOT close the shared dual-scope `cleo.db` handle (co-owned by the tasks + // domain). After reset, getBrainDb re-derives a fresh brain drizzle wrapper + // bound to the SAME live shared handle — proving the singleton was cleared + // (re-init ran) while the shared connection stayed alive. resetBrainDbState(); const db2 = await getBrainDb(); expect(db2).toBeDefined(); + // A new drizzle wrapper instance was produced (the singleton was cleared). + expect(Object.is(db2, db1)).toBe(false); const nativeDb2 = getBrainNativeDb(); - // Use Object.is() to compare — avoids Vitest serializing closed DatabaseSync objects - // which throws "database is not open" during pretty-print of assertion diffs. - expect(Object.is(nativeDb2, nativeDb1)).toBe(false); + expect(nativeDb2).not.toBeNull(); + // The underlying shared dual-scope handle is the SAME and still open. + expect(Object.is(nativeDb2, nativeDb1)).toBe(true); + expect(nativeDb2?.isOpen).toBe(true); }); it('resetBrainDbState is safe to call multiple times', async () => { diff --git a/packages/core/src/store/memory-sqlite.ts b/packages/core/src/store/memory-sqlite.ts index 0892b123f..f2ca227d0 100644 --- a/packages/core/src/store/memory-sqlite.ts +++ b/packages/core/src/store/memory-sqlite.ts @@ -246,14 +246,28 @@ function establishLegacyBrainSchema( // `brain_embeddings` is a vec0 virtual table once the extension is loaded; // DROP TABLE handles both regular and virtual tables when sqlite-vec is // present. Disable FKs during the drop so cross-table references do not - // block the teardown. + // block the teardown — then RESTORE the prior pragma state (the dual-scope + // pragma SSoT enables foreign_keys; leaving it OFF would break the + // idempotent-pragma contract, T10314). + const fkRow = nativeDb.prepare('PRAGMA foreign_keys').get() as + | { foreign_keys?: number } + | undefined; + const fkWasOn = fkRow?.foreign_keys === 1; nativeDb.exec('PRAGMA foreign_keys=OFF'); - for (const table of CONSOLIDATED_BRAIN_TABLES) { - try { - nativeDb.exec(`DROP TABLE IF EXISTS \`${table}\``); - } catch (err) { - log.warn({ table, err }, 'Failed to drop consolidated brain table during legacy rebuild.'); + try { + for (const table of CONSOLIDATED_BRAIN_TABLES) { + try { + nativeDb.exec(`DROP TABLE IF EXISTS \`${table}\``); + } catch (err) { + log.warn( + { table, err }, + 'Failed to drop consolidated brain table during legacy rebuild.', + ); + } } + } finally { + // Restore the pragma to its pre-drop state (ON under the dual-scope SSoT). + nativeDb.exec(`PRAGMA foreign_keys=${fkWasOn ? 'ON' : 'OFF'}`); } log.debug( { count: CONSOLIDATED_BRAIN_TABLES.length }, From e0fdb2b536976ed7d9a224834003f1b3962853cc Mon Sep 17 00:00:00 2001 From: kryptobaseddev Date: Mon, 1 Jun 2026 01:32:16 -0700 Subject: [PATCH 6/6] fix(T11522): retarget brain health/scaffold probes to cleo.db (E6-L2) - project-health.ts: BRAIN_DB -> cleo.db (dbs.brain probe targets the shared consolidated DB); collapse DB_EXPECTED_VERSIONS to single cleo.db floor - scaffold/ensure-dirs.ts ensureBrainDb: idempotency probe via getBrainDbPath (cleo.db) so re-scaffold reports skipped not always-created - scaffold/project-detection.ts checkBrainDb -> cleo.db - scaffold-characterization tests updated to cleo.db (mirrors L1) Co-Authored-By: Claude Opus 4.8 --- .../scaffold-characterization.test.ts | 12 ++++---- packages/core/src/scaffold/ensure-dirs.ts | 29 +++++++++++++------ .../core/src/scaffold/project-detection.ts | 14 +++++---- packages/core/src/system/project-health.ts | 18 ++++++++---- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/packages/core/src/__tests__/scaffold-characterization.test.ts b/packages/core/src/__tests__/scaffold-characterization.test.ts index cbc967fef..b587f97ce 100644 --- a/packages/core/src/__tests__/scaffold-characterization.test.ts +++ b/packages/core/src/__tests__/scaffold-characterization.test.ts @@ -409,16 +409,18 @@ describe('characterization: ensureBrainDb', () => { else delete process.env['CLEO_DIR']; }); - it('returns action=skipped when brain.db already exists', async () => { - writeFileSync(join(tmpDir, '.cleo', 'brain.db'), 'SQLite format 3'); + // E6-L2 (T11522): ensureBrainDb now probes the consolidated `cleo.db` + // (getBrainDb → openDualScopeDb('project')), not a standalone `brain.db`. + it('returns action=skipped when cleo.db already exists', async () => { + writeFileSync(join(tmpDir, '.cleo', 'cleo.db'), 'SQLite format 3'); const result = await ensureBrainDb(tmpDir); expect(result.action).toBe('skipped'); }); - it('result path points to brain.db', async () => { - writeFileSync(join(tmpDir, '.cleo', 'brain.db'), 'SQLite format 3'); + it('result path points to cleo.db', async () => { + writeFileSync(join(tmpDir, '.cleo', 'cleo.db'), 'SQLite format 3'); const result = await ensureBrainDb(tmpDir); - expect(result.path).toContain('brain.db'); + expect(result.path).toContain('cleo.db'); }); }); diff --git a/packages/core/src/scaffold/ensure-dirs.ts b/packages/core/src/scaffold/ensure-dirs.ts index f02fe51ba..1ef28e5c2 100644 --- a/packages/core/src/scaffold/ensure-dirs.ts +++ b/packages/core/src/scaffold/ensure-dirs.ts @@ -206,19 +206,27 @@ export async function ensureSqliteDb(projectRoot: string): Promise { - const cleoDir = resolveScaffoldCleoDir(projectRoot); - const dbPath = join(cleoDir, 'brain.db'); + const { getBrainDbPath, getBrainDb, getBrainNativeDb } = await import( + '../store/memory-sqlite.js' + ); + // E6-L2: getBrainDbPath resolves to the consolidated `cleo.db`. + const dbPath = getBrainDbPath(projectRoot); if (existsSync(dbPath)) { try { - const { getBrainNativeDb } = await import('../store/memory-sqlite.js'); const nativeDb = getBrainNativeDb(); if (nativeDb) { const { ensureFts5Tables } = await import('../memory/brain-search.js'); @@ -227,11 +235,10 @@ export async function ensureBrainDb(projectRoot: string): Promise> = { - // E6-L1 (T11521): TASKS_DB now resolves to `cleo.db`; the consolidated - // project schema is tracked by the `drizzle-cleo-project` migration set. + // E6-L1 (T11521) / E6-L2 (T11522): TASKS_DB and BRAIN_DB both resolve to + // `cleo.db`. The consolidated project schema is tracked by the + // `drizzle-cleo-project` migration set (3 entries); the brain domain adds its + // own `drizzle-brain` entries to the SAME `__drizzle_migrations` journal, so + // the live count is strictly ≥ 3. We keep the conservative cleo-project floor + // (3) here — drift fires only when the count drops BELOW it, which the merged + // journal never does. (TASKS_DB === BRAIN_DB === 'cleo.db'.) [TASKS_DB]: 3, - [BRAIN_DB]: 14, [NEXUS_DB]: 3, [SIGNALDOCK_DB]: 1, [CONDUIT_DB]: 1,