From 0d2880d57d94bf1b19fcae02336af4c2be2e6242 Mon Sep 17 00:00:00 2001 From: Joakim Ohlander Date: Tue, 24 Feb 2026 15:32:37 +0100 Subject: [PATCH 1/3] Optimize upsertAssetBatch and remove unused getAssetStats - upsertAssetBatch: use RETURNING clause to collect rows directly from INSERT instead of separate post-upsert batch SELECT queries - Remove getAssetStats() and /asset-stats endpoint (no production consumers) - Fold totalAssets and blueprintCount into getStats() for reuse - Track blueprintCount incrementally in memory-index add/remove hooks Closes #57 tracks follow-up for refreshStatsCache timer refactoring. Co-Authored-By: Claude Opus 4.6 --- src/blueprint-test.js | 10 +++---- src/perf-test.js | 2 +- src/service/api.js | 8 +----- src/service/database.js | 52 ++++++++----------------------------- src/service/index.js | 7 +++-- src/service/memory-index.js | 36 ++++++++----------------- src/service/query-worker.js | 3 +-- src/wsl-migration.test.js | 8 +++--- 8 files changed, 37 insertions(+), 89 deletions(-) diff --git a/src/blueprint-test.js b/src/blueprint-test.js index facb874..cbb5d6e 100644 --- a/src/blueprint-test.js +++ b/src/blueprint-test.js @@ -323,7 +323,7 @@ describe('findTypeByName with Blueprint assets', () => { }); }); -describe('getAssetStats with Blueprint info', () => { +describe('getStats includes asset counts', () => { let db; before(() => { @@ -343,11 +343,9 @@ describe('getAssetStats with Blueprint info', () => { teardown(); }); - it('reports asset class breakdown', () => { - const stats = db.getAssetStats(); - assert.equal(stats.total, 3); + it('reports totalAssets and blueprintCount in getStats', () => { + const stats = db.getStats(); + assert.equal(stats.totalAssets, 3); assert.equal(stats.blueprintCount, 2); - assert.ok(stats.byAssetClass.some(r => r.asset_class === 'BlueprintGeneratedClass' && r.count === 2)); - assert.ok(stats.byAssetClass.some(r => r.asset_class === 'Material' && r.count === 1)); }); }); diff --git a/src/perf-test.js b/src/perf-test.js index 2cbddab..178f0e6 100644 --- a/src/perf-test.js +++ b/src/perf-test.js @@ -184,7 +184,7 @@ async function main() { // Asset searches console.log('\n --- Asset Searches ---'); await runBenchmark('/find-asset exact', '/find-asset?name=BP_&fuzzy=true&maxResults=20'); - await runBenchmark('/asset-stats', '/asset-stats'); + // /asset-stats removed — asset counts now in /stats // Grep (trigram index) console.log('\n --- Content Search (Grep) ---'); diff --git a/src/service/api.js b/src/service/api.js index b460d4f..652e0e1 100644 --- a/src/service/api.js +++ b/src/service/api.js @@ -1385,13 +1385,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n } }); - app.get('/asset-stats', (req, res) => { - try { - res.json((memoryIndex && memoryIndex.isLoaded) ? memoryIndex.getAssetStats() : database.getAssetStats()); - } catch (err) { - res.status(500).json({ error: err.message }); - } - }); + // /asset-stats removed — asset counts folded into /stats (totalAssets, blueprintCount) // --- Query Analytics --- diff --git a/src/service/database.js b/src/service/database.js index 0f51ab7..0a19ba9 100644 --- a/src/service/database.js +++ b/src/service/database.js @@ -1611,6 +1611,8 @@ export class IndexDatabase { const totalFiles = this.db.prepare("SELECT COUNT(*) as count FROM files WHERE language != 'asset'").get().count; const totalTypes = this.db.prepare('SELECT COUNT(*) as count FROM types').get().count; const totalMembers = this.db.prepare('SELECT COUNT(*) as count FROM members').get().count; + const totalAssets = this.db.prepare('SELECT COUNT(*) as count FROM assets').get().count; + const blueprintCount = this.db.prepare('SELECT COUNT(*) as count FROM assets WHERE parent_class IS NOT NULL').get().count; const kindCounts = this.db.prepare(` SELECT kind, COUNT(*) as count FROM types GROUP BY kind @@ -1636,6 +1638,8 @@ export class IndexDatabase { totalFiles, totalTypes, totalMembers, + totalAssets, + blueprintCount, byKind: {}, byMemberKind: {}, byLanguage: {}, @@ -1772,31 +1776,20 @@ export class IndexDatabase { mtime = excluded.mtime, asset_class = excluded.asset_class, parent_class = excluded.parent_class + RETURNING id, path, name, content_path, folder, project, extension, mtime, asset_class, parent_class `); const insertMany = this.db.transaction((items) => { + const results = []; for (const item of items) { - stmt.run(item.path, item.name, item.contentPath, item.folder, item.project, item.extension, item.mtime, + const row = stmt.get(item.path, item.name, item.contentPath, item.folder, item.project, item.extension, item.mtime, item.assetClass || null, item.parentClass || null); + results.push(row); } + return results; }); - insertMany(assets); - - // Batch fetch all upserted rows (1 query instead of N individual SELECTs) - const paths = assets.map(a => a.path); - const results = []; - const chunkSize = 900; - for (let i = 0; i < paths.length; i += chunkSize) { - const chunk = paths.slice(i, i + chunkSize); - const ph = chunk.map(() => '?').join(','); - const rows = this.db.prepare(` - SELECT id, path, name, content_path, folder, project, extension, mtime, asset_class, parent_class - FROM assets WHERE path IN (${ph}) - `).all(...chunk); - results.push(...rows); - } - return results; + return insertMany(assets); } deleteAsset(path) { @@ -1962,29 +1955,6 @@ export class IndexDatabase { .sort((a, b) => a.path.localeCompare(b.path)); } - getAssetStats() { - const total = this.db.prepare('SELECT COUNT(*) as count FROM assets').get().count; - - const byProject = this.db.prepare(` - SELECT project, COUNT(*) as count FROM assets GROUP BY project - `).all(); - - const byExtension = this.db.prepare(` - SELECT extension, COUNT(*) as count FROM assets GROUP BY extension - `).all(); - - const byAssetClass = this.db.prepare(` - SELECT COALESCE(asset_class, 'Unknown') as asset_class, COUNT(*) as count - FROM assets GROUP BY asset_class ORDER BY count DESC - `).all(); - - const blueprintCount = this.db.prepare(` - SELECT COUNT(*) as count FROM assets WHERE parent_class IS NOT NULL - `).get().count; - - return { total, byProject, byExtension, byAssetClass, blueprintCount }; - } - // --- Trigram index methods --- upsertFileContent(fileId, compressedContent, hash) { @@ -2383,7 +2353,7 @@ export class IndexDatabase { // Wrap key methods with slow-query timing and analytics logging const methodsToTime = [ 'findTypeByName', 'findChildrenOf', 'findMember', 'findFileByName', - 'findAssetByName', 'getStats', 'getAssetStats', 'upsertAssetBatch', + 'findAssetByName', 'getStats', 'upsertAssetBatch', ]; for (const method of methodsToTime) { const original = IndexDatabase.prototype[method]; diff --git a/src/service/index.js b/src/service/index.js index c57d4c2..f179908 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -292,7 +292,6 @@ class UnrealIndexService { printIndexSummary() { const stats = this.database.getStats(); - const assetStats = this.database.getAssetStats(); console.log('--- Index Summary ---'); for (const [lang, langStats] of Object.entries(stats.byLanguage)) { @@ -304,9 +303,9 @@ class UnrealIndexService { } } - if (assetStats.total > 0) { - const bpCount = assetStats.blueprintCount || 0; - console.log(` assets: ${assetStats.total} files (${bpCount} with class hierarchy)`); + if (stats.totalAssets > 0) { + const bpCount = stats.blueprintCount || 0; + console.log(` assets: ${stats.totalAssets} files (${bpCount} with class hierarchy)`); } const kindEntries = Object.entries(stats.byKind); diff --git a/src/service/memory-index.js b/src/service/memory-index.js index f57894f..48c131e 100644 --- a/src/service/memory-index.js +++ b/src/service/memory-index.js @@ -73,7 +73,7 @@ export class MemoryIndex { // Stats counters this._stats = { - totalFiles: 0, totalTypes: 0, totalMembers: 0, totalAssets: 0, + totalFiles: 0, totalTypes: 0, totalMembers: 0, totalAssets: 0, blueprintCount: 0, byKind: {}, byMemberKind: {}, byLanguage: {}, projects: {} }; @@ -348,6 +348,7 @@ export class MemoryIndex { this._addToMultiMap(this.assetsByFolder, rec.folder, rec.id); this._addToMultiMap(this.assetsByProject, rec.project, rec.id); this._stats.totalAssets++; + if (rec.parentClass) this._stats.blueprintCount++; } _removeAssetRecord(assetId) { @@ -361,6 +362,7 @@ export class MemoryIndex { this._removeFromMultiMap(this.assetsByFolder, rec.folder, assetId); this._removeFromMultiMap(this.assetsByProject, rec.project, assetId); this._stats.totalAssets--; + if (rec.parentClass) this._stats.blueprintCount--; } _buildTrigramIndexes() { @@ -1376,6 +1378,8 @@ export class MemoryIndex { totalFiles: this._stats.totalFiles, totalTypes: this._stats.totalTypes, totalMembers: this._stats.totalMembers, + totalAssets: this._stats.totalAssets, + blueprintCount: this._stats.blueprintCount, byKind: { ...this._stats.byKind }, byMemberKind: { ...this._stats.byMemberKind }, byLanguage: { ...this._stats.byLanguage }, @@ -1383,30 +1387,6 @@ export class MemoryIndex { }; } - getAssetStats() { - const byProject = new Map(); - const byExtension = new Map(); - const byAssetClass = new Map(); - let blueprintCount = 0; - - for (const [, a] of this.assetsById) { - byProject.set(a.project, (byProject.get(a.project) || 0) + 1); - byExtension.set(a.extension, (byExtension.get(a.extension) || 0) + 1); - const cls = a.assetClass || 'Unknown'; - byAssetClass.set(cls, (byAssetClass.get(cls) || 0) + 1); - if (a.parentClass) blueprintCount++; - } - - return { - total: this.assetsById.size, - byProject: [...byProject.entries()].map(([project, count]) => ({ project, count })), - byExtension: [...byExtension.entries()].map(([extension, count]) => ({ extension, count })), - byAssetClass: [...byAssetClass.entries()].map(([asset_class, count]) => ({ asset_class, count })) - .sort((a, b) => b.count - a.count), - blueprintCount - }; - } - // --- Grep support methods (replaces DB queries for grep handler) --- /** @@ -1484,6 +1464,7 @@ export class MemoryIndex { totalTypes: this.typesById.size, totalMembers: this.membersById.size, totalAssets: this.assetsById.size, + blueprintCount: 0, byKind: {}, byMemberKind: {}, byLanguage: {}, @@ -1512,6 +1493,11 @@ export class MemoryIndex { for (const [, m] of this.membersById) { this._stats.byMemberKind[m.memberKind] = (this._stats.byMemberKind[m.memberKind] || 0) + 1; } + + // Asset stats + for (const [, a] of this.assetsById) { + if (a.parentClass) this._stats.blueprintCount++; + } } // --- Phase 6: Ingest sync hooks --- diff --git a/src/service/query-worker.js b/src/service/query-worker.js index 2d11c4c..b36208c 100644 --- a/src/service/query-worker.js +++ b/src/service/query-worker.js @@ -17,8 +17,7 @@ const ALLOWED_METHODS = new Set([ 'listModules', 'browseAssetFolder', 'listAssetFolders', - 'getStats', - 'getAssetStats' + 'getStats' ]); parentPort.on('message', (msg) => { diff --git a/src/wsl-migration.test.js b/src/wsl-migration.test.js index 51914a7..92aec58 100644 --- a/src/wsl-migration.test.js +++ b/src/wsl-migration.test.js @@ -318,10 +318,12 @@ describe('POST /internal/ingest — assets', () => { assert.equal(data.results[0].name, 'BP_Player'); }); - it('should return asset stats', async () => { - const { status, data } = await fetchJson(`${BASE}/asset-stats`); + it('should include asset counts in stats', async () => { + const { status, data } = await fetchJson(`${BASE}/stats`); assert.equal(status, 200); - assert.ok(data.total >= 2, `expected >= 2 assets, got ${data.total}`); + // totalAssets and blueprintCount are now part of getStats() + assert.ok('totalAssets' in data, 'stats should include totalAssets field'); + assert.ok('blueprintCount' in data, 'stats should include blueprintCount field'); }); }); From dec3023f94dcd7d1b4869119b40746b3fe1e30b4 Mon Sep 17 00:00:00 2001 From: Joakim Ohlander Date: Tue, 24 Feb 2026 19:02:08 +0100 Subject: [PATCH 2/3] Refactor refreshStatsCache: replace 30s timer with lazy on-demand caching Replace the periodic 30s setInterval with a lazy getStatsCache() that computes stats on-demand with a 30s TTL. Decouple computeInheritanceDepth from stats caching into a debounced post-ingest trigger (5s debounce). Closes #57 Co-Authored-By: Claude Opus 4.6 --- src/service/api.js | 67 ++++++++++++++++++++++++--------------- src/wsl-migration.test.js | 4 +-- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/service/api.js b/src/service/api.js index 652e0e1..4130426 100644 --- a/src/service/api.js +++ b/src/service/api.js @@ -736,6 +736,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n // Flag inheritance depth for recomputation after new types are ingested if (processed > 0) { database.setMetadata('depthComputeNeeded', true); + scheduleDepthRecompute(); grepCache.invalidate(); if (memoryIndex) memoryIndex.invalidateInheritanceCache(); } @@ -960,18 +961,13 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n } }); - // Stats cache — refreshed on a timer, never blocks request handling + // Stats cache — lazy on-demand with TTL, never blocks request handling let statsCache = null; + let statsCacheTime = 0; + const STATS_TTL_MS = 30000; function refreshStatsCache() { try { - // Recompute inheritance depth if flagged (debounced via timer) - if (database.getMetadata('depthComputeNeeded')) { - const t = performance.now(); - const count = database.computeInheritanceDepth(); - console.log(`[Stats] inheritance depth recomputed: ${count} types (${(performance.now() - t).toFixed(0)}ms)`); - } - const stats = (memoryIndex && memoryIndex.isLoaded) ? memoryIndex.getStats() : database.getStats(); const lastBuild = database.getMetadata('lastBuild'); const indexStatus = database.getAllIndexStatus(); @@ -990,18 +986,40 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n } } - // Refresh stats every 30 seconds in the background - refreshStatsCache(); - app._statsInterval = setInterval(refreshStatsCache, 30000); + function getStatsCache() { + const now = Date.now(); + if (statsCache && (now - statsCacheTime) < STATS_TTL_MS) { + return statsCache; + } + refreshStatsCache(); + statsCacheTime = now; + return statsCache; + } + + // Debounced inheritance depth recomputation — triggered after ingest + let depthDebounceTimer = null; + const DEPTH_DEBOUNCE_MS = 5000; + + function scheduleDepthRecompute() { + if (depthDebounceTimer) clearTimeout(depthDebounceTimer); + depthDebounceTimer = setTimeout(() => { + depthDebounceTimer = null; + try { + if (database.getMetadata('depthComputeNeeded')) { + const t = performance.now(); + const count = database.computeInheritanceDepth(); + console.log(`[Stats] inheritance depth recomputed: ${count} types (${(performance.now() - t).toFixed(0)}ms)`); + } + } catch (err) { + console.error(`[${new Date().toISOString()}] [Stats] depth recompute failed:`, err.message); + } + }, DEPTH_DEBOUNCE_MS); + app._depthDebounceTimer = depthDebounceTimer; + } app.get('/stats', (req, res) => { - if (statsCache) { - return res.json(statsCache); - } - // Fallback: compute on demand if cache hasn't been populated yet try { - refreshStatsCache(); - res.json(statsCache); + res.json(getStatsCache()); } catch (err) { res.status(500).json({ error: err.message }); } @@ -1160,17 +1178,16 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n app.get('/summary', (req, res) => { try { - const stats = statsCache || database.getStats(); - const lastBuild = database.getMetadata('lastBuild'); - const indexStatus = statsCache?.indexStatus || database.getAllIndexStatus(); + const cached = getStatsCache(); + const lastBuild = cached.lastBuild; res.json({ generatedAt: lastBuild?.timestamp || null, - stats, - projects: Object.keys(stats.projects || {}), - languages: Object.keys(stats.byLanguage || {}), + stats: cached, + projects: Object.keys(cached.projects || {}), + languages: Object.keys(cached.byLanguage || {}), buildTimeMs: lastBuild?.buildTimeMs || null, - indexStatus + indexStatus: cached.indexStatus }); } catch (err) { res.status(500).json({ error: err.message }); @@ -1500,7 +1517,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n const duration = ((performance.now() - start) / 1000).toFixed(1); console.log(`[${new Date().toISOString()}] [NameTrigram] Build complete in ${duration}s`); - refreshStatsCache(); + statsCacheTime = 0; // force refresh on next stats request res.json({ message: 'Name trigram index built successfully', types: result.types, diff --git a/src/wsl-migration.test.js b/src/wsl-migration.test.js index 92aec58..7a51bb4 100644 --- a/src/wsl-migration.test.js +++ b/src/wsl-migration.test.js @@ -85,8 +85,8 @@ before(async () => { }); after(() => { - // Clear all intervals to prevent process hanging - if (app?._statsInterval) clearInterval(app._statsInterval); + // Clear timers to prevent process hanging + if (app?._depthDebounceTimer) clearTimeout(app._depthDebounceTimer); if (app?._watcherPruneInterval) clearInterval(app._watcherPruneInterval); if (server) server.close(); if (database) database.close(); From 548c38aff79e0d70b586ab84a6db7f1805d38fab Mon Sep 17 00:00:00 2001 From: Joakim Ohlander Date: Tue, 24 Feb 2026 19:07:18 +0100 Subject: [PATCH 3/3] Add null guard for stats endpoints and clear stale depth timer ref Handle the case where getStatsCache() returns null (e.g. DB unavailable at cold start) by returning 503 from /stats and /summary. Also clear app._depthDebounceTimer when the debounce callback fires to keep the exposed handle in sync with the closure variable. Co-Authored-By: Claude Opus 4.6 --- src/service/api.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/service/api.js b/src/service/api.js index 4130426..4059340 100644 --- a/src/service/api.js +++ b/src/service/api.js @@ -1004,6 +1004,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n if (depthDebounceTimer) clearTimeout(depthDebounceTimer); depthDebounceTimer = setTimeout(() => { depthDebounceTimer = null; + app._depthDebounceTimer = null; try { if (database.getMetadata('depthComputeNeeded')) { const t = performance.now(); @@ -1019,7 +1020,9 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n app.get('/stats', (req, res) => { try { - res.json(getStatsCache()); + const data = getStatsCache(); + if (!data) return res.status(503).json({ error: 'Stats not yet available' }); + res.json(data); } catch (err) { res.status(500).json({ error: err.message }); } @@ -1179,6 +1182,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n app.get('/summary', (req, res) => { try { const cached = getStatsCache(); + if (!cached) return res.status(503).json({ error: 'Stats not yet available' }); const lastBuild = cached.lastBuild; res.json({