Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions src/blueprint-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ describe('findTypeByName with Blueprint assets', () => {
});
});

describe('getAssetStats with Blueprint info', () => {
describe('getStats includes asset counts', () => {
let db;

before(() => {
Expand All @@ -343,11 +343,9 @@ describe('getAssetStats with Blueprint info', () => {
teardown();
});

it('reports asset class breakdown', () => {
const stats = db.getAssetStats();
assert.equal(stats.total, 3);
it('reports totalAssets and blueprintCount in getStats', () => {
const stats = db.getStats();
assert.equal(stats.totalAssets, 3);
assert.equal(stats.blueprintCount, 2);
assert.ok(stats.byAssetClass.some(r => r.asset_class === 'BlueprintGeneratedClass' && r.count === 2));
assert.ok(stats.byAssetClass.some(r => r.asset_class === 'Material' && r.count === 1));
});
});
2 changes: 1 addition & 1 deletion src/perf-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ async function main() {
// Asset searches
console.log('\n --- Asset Searches ---');
await runBenchmark('/find-asset exact', '/find-asset?name=BP_&fuzzy=true&maxResults=20');
await runBenchmark('/asset-stats', '/asset-stats');
// /asset-stats removed — asset counts now in /stats

// Grep (trigram index)
console.log('\n --- Content Search (Grep) ---');
Expand Down
79 changes: 47 additions & 32 deletions src/service/api.js
Original file line number Diff line number Diff line change
Expand Up @@ -736,6 +736,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
// Flag inheritance depth for recomputation after new types are ingested
if (processed > 0) {
database.setMetadata('depthComputeNeeded', true);
scheduleDepthRecompute();
grepCache.invalidate();
if (memoryIndex) memoryIndex.invalidateInheritanceCache();
}
Expand Down Expand Up @@ -960,18 +961,13 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
}
});

// Stats cache — refreshed on a timer, never blocks request handling
// Stats cache — lazy on-demand with TTL, never blocks request handling
let statsCache = null;
let statsCacheTime = 0;
const STATS_TTL_MS = 30000;

function refreshStatsCache() {
try {
// Recompute inheritance depth if flagged (debounced via timer)
if (database.getMetadata('depthComputeNeeded')) {
const t = performance.now();
const count = database.computeInheritanceDepth();
console.log(`[Stats] inheritance depth recomputed: ${count} types (${(performance.now() - t).toFixed(0)}ms)`);
}

const stats = (memoryIndex && memoryIndex.isLoaded) ? memoryIndex.getStats() : database.getStats();
const lastBuild = database.getMetadata('lastBuild');
const indexStatus = database.getAllIndexStatus();
Expand All @@ -990,18 +986,43 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
}
}

// Refresh stats every 30 seconds in the background
refreshStatsCache();
app._statsInterval = setInterval(refreshStatsCache, 30000);
function getStatsCache() {
const now = Date.now();
if (statsCache && (now - statsCacheTime) < STATS_TTL_MS) {
return statsCache;
}
refreshStatsCache();
statsCacheTime = now;
return statsCache;
}

// Debounced inheritance depth recomputation — triggered after ingest
let depthDebounceTimer = null;
const DEPTH_DEBOUNCE_MS = 5000;

function scheduleDepthRecompute() {
if (depthDebounceTimer) clearTimeout(depthDebounceTimer);
depthDebounceTimer = setTimeout(() => {
depthDebounceTimer = null;
app._depthDebounceTimer = null;
try {
if (database.getMetadata('depthComputeNeeded')) {
const t = performance.now();
const count = database.computeInheritanceDepth();
console.log(`[Stats] inheritance depth recomputed: ${count} types (${(performance.now() - t).toFixed(0)}ms)`);
}
} catch (err) {
console.error(`[${new Date().toISOString()}] [Stats] depth recompute failed:`, err.message);
}
}, DEPTH_DEBOUNCE_MS);
app._depthDebounceTimer = depthDebounceTimer;
}

app.get('/stats', (req, res) => {
if (statsCache) {
return res.json(statsCache);
}
// Fallback: compute on demand if cache hasn't been populated yet
try {
refreshStatsCache();
res.json(statsCache);
const data = getStatsCache();
if (!data) return res.status(503).json({ error: 'Stats not yet available' });
res.json(data);
} catch (err) {
res.status(500).json({ error: err.message });
}
Expand Down Expand Up @@ -1160,17 +1181,17 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n

app.get('/summary', (req, res) => {
try {
const stats = statsCache || database.getStats();
const lastBuild = database.getMetadata('lastBuild');
const indexStatus = statsCache?.indexStatus || database.getAllIndexStatus();
const cached = getStatsCache();
if (!cached) return res.status(503).json({ error: 'Stats not yet available' });
const lastBuild = cached.lastBuild;

res.json({
generatedAt: lastBuild?.timestamp || null,
stats,
projects: Object.keys(stats.projects || {}),
languages: Object.keys(stats.byLanguage || {}),
stats: cached,
projects: Object.keys(cached.projects || {}),
languages: Object.keys(cached.byLanguage || {}),
buildTimeMs: lastBuild?.buildTimeMs || null,
indexStatus
indexStatus: cached.indexStatus
});
} catch (err) {
res.status(500).json({ error: err.message });
Expand Down Expand Up @@ -1385,13 +1406,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
}
});

app.get('/asset-stats', (req, res) => {
try {
res.json((memoryIndex && memoryIndex.isLoaded) ? memoryIndex.getAssetStats() : database.getAssetStats());
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// /asset-stats removed — asset counts folded into /stats (totalAssets, blueprintCount)

// --- Query Analytics ---

Expand Down Expand Up @@ -1506,7 +1521,7 @@ export function createApi(database, indexer, queryPool = null, { zoektClient = n
const duration = ((performance.now() - start) / 1000).toFixed(1);
console.log(`[${new Date().toISOString()}] [NameTrigram] Build complete in ${duration}s`);

refreshStatsCache();
statsCacheTime = 0; // force refresh on next stats request
res.json({
message: 'Name trigram index built successfully',
types: result.types,
Expand Down
52 changes: 11 additions & 41 deletions src/service/database.js
Original file line number Diff line number Diff line change
Expand Up @@ -1611,6 +1611,8 @@ export class IndexDatabase {
const totalFiles = this.db.prepare("SELECT COUNT(*) as count FROM files WHERE language != 'asset'").get().count;
const totalTypes = this.db.prepare('SELECT COUNT(*) as count FROM types').get().count;
const totalMembers = this.db.prepare('SELECT COUNT(*) as count FROM members').get().count;
const totalAssets = this.db.prepare('SELECT COUNT(*) as count FROM assets').get().count;
const blueprintCount = this.db.prepare('SELECT COUNT(*) as count FROM assets WHERE parent_class IS NOT NULL').get().count;

const kindCounts = this.db.prepare(`
SELECT kind, COUNT(*) as count FROM types GROUP BY kind
Expand All @@ -1636,6 +1638,8 @@ export class IndexDatabase {
totalFiles,
totalTypes,
totalMembers,
totalAssets,
blueprintCount,
byKind: {},
byMemberKind: {},
byLanguage: {},
Expand Down Expand Up @@ -1772,31 +1776,20 @@ export class IndexDatabase {
mtime = excluded.mtime,
asset_class = excluded.asset_class,
parent_class = excluded.parent_class
RETURNING id, path, name, content_path, folder, project, extension, mtime, asset_class, parent_class
`);

const insertMany = this.db.transaction((items) => {
const results = [];
for (const item of items) {
stmt.run(item.path, item.name, item.contentPath, item.folder, item.project, item.extension, item.mtime,
const row = stmt.get(item.path, item.name, item.contentPath, item.folder, item.project, item.extension, item.mtime,
item.assetClass || null, item.parentClass || null);
results.push(row);
}
return results;
});

insertMany(assets);

// Batch fetch all upserted rows (1 query instead of N individual SELECTs)
const paths = assets.map(a => a.path);
const results = [];
const chunkSize = 900;
for (let i = 0; i < paths.length; i += chunkSize) {
const chunk = paths.slice(i, i + chunkSize);
const ph = chunk.map(() => '?').join(',');
const rows = this.db.prepare(`
SELECT id, path, name, content_path, folder, project, extension, mtime, asset_class, parent_class
FROM assets WHERE path IN (${ph})
`).all(...chunk);
results.push(...rows);
}
return results;
return insertMany(assets);
}

deleteAsset(path) {
Expand Down Expand Up @@ -1962,29 +1955,6 @@ export class IndexDatabase {
.sort((a, b) => a.path.localeCompare(b.path));
}

getAssetStats() {
const total = this.db.prepare('SELECT COUNT(*) as count FROM assets').get().count;

const byProject = this.db.prepare(`
SELECT project, COUNT(*) as count FROM assets GROUP BY project
`).all();

const byExtension = this.db.prepare(`
SELECT extension, COUNT(*) as count FROM assets GROUP BY extension
`).all();

const byAssetClass = this.db.prepare(`
SELECT COALESCE(asset_class, 'Unknown') as asset_class, COUNT(*) as count
FROM assets GROUP BY asset_class ORDER BY count DESC
`).all();

const blueprintCount = this.db.prepare(`
SELECT COUNT(*) as count FROM assets WHERE parent_class IS NOT NULL
`).get().count;

return { total, byProject, byExtension, byAssetClass, blueprintCount };
}

// --- Trigram index methods ---

upsertFileContent(fileId, compressedContent, hash) {
Expand Down Expand Up @@ -2383,7 +2353,7 @@ export class IndexDatabase {
// Wrap key methods with slow-query timing and analytics logging
const methodsToTime = [
'findTypeByName', 'findChildrenOf', 'findMember', 'findFileByName',
'findAssetByName', 'getStats', 'getAssetStats', 'upsertAssetBatch',
'findAssetByName', 'getStats', 'upsertAssetBatch',
];
for (const method of methodsToTime) {
const original = IndexDatabase.prototype[method];
Expand Down
7 changes: 3 additions & 4 deletions src/service/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,6 @@ class UnrealIndexService {

printIndexSummary() {
const stats = this.database.getStats();
const assetStats = this.database.getAssetStats();

console.log('--- Index Summary ---');
for (const [lang, langStats] of Object.entries(stats.byLanguage)) {
Expand All @@ -304,9 +303,9 @@ class UnrealIndexService {
}
}

if (assetStats.total > 0) {
const bpCount = assetStats.blueprintCount || 0;
console.log(` assets: ${assetStats.total} files (${bpCount} with class hierarchy)`);
if (stats.totalAssets > 0) {
const bpCount = stats.blueprintCount || 0;
console.log(` assets: ${stats.totalAssets} files (${bpCount} with class hierarchy)`);
}

const kindEntries = Object.entries(stats.byKind);
Expand Down
36 changes: 11 additions & 25 deletions src/service/memory-index.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export class MemoryIndex {

// Stats counters
this._stats = {
totalFiles: 0, totalTypes: 0, totalMembers: 0, totalAssets: 0,
totalFiles: 0, totalTypes: 0, totalMembers: 0, totalAssets: 0, blueprintCount: 0,
byKind: {}, byMemberKind: {}, byLanguage: {}, projects: {}
};

Expand Down Expand Up @@ -348,6 +348,7 @@ export class MemoryIndex {
this._addToMultiMap(this.assetsByFolder, rec.folder, rec.id);
this._addToMultiMap(this.assetsByProject, rec.project, rec.id);
this._stats.totalAssets++;
if (rec.parentClass) this._stats.blueprintCount++;
}

_removeAssetRecord(assetId) {
Expand All @@ -361,6 +362,7 @@ export class MemoryIndex {
this._removeFromMultiMap(this.assetsByFolder, rec.folder, assetId);
this._removeFromMultiMap(this.assetsByProject, rec.project, assetId);
this._stats.totalAssets--;
if (rec.parentClass) this._stats.blueprintCount--;
}

_buildTrigramIndexes() {
Expand Down Expand Up @@ -1376,37 +1378,15 @@ export class MemoryIndex {
totalFiles: this._stats.totalFiles,
totalTypes: this._stats.totalTypes,
totalMembers: this._stats.totalMembers,
totalAssets: this._stats.totalAssets,
blueprintCount: this._stats.blueprintCount,
byKind: { ...this._stats.byKind },
byMemberKind: { ...this._stats.byMemberKind },
byLanguage: { ...this._stats.byLanguage },
projects: { ...this._stats.projects }
};
}

getAssetStats() {
const byProject = new Map();
const byExtension = new Map();
const byAssetClass = new Map();
let blueprintCount = 0;

for (const [, a] of this.assetsById) {
byProject.set(a.project, (byProject.get(a.project) || 0) + 1);
byExtension.set(a.extension, (byExtension.get(a.extension) || 0) + 1);
const cls = a.assetClass || 'Unknown';
byAssetClass.set(cls, (byAssetClass.get(cls) || 0) + 1);
if (a.parentClass) blueprintCount++;
}

return {
total: this.assetsById.size,
byProject: [...byProject.entries()].map(([project, count]) => ({ project, count })),
byExtension: [...byExtension.entries()].map(([extension, count]) => ({ extension, count })),
byAssetClass: [...byAssetClass.entries()].map(([asset_class, count]) => ({ asset_class, count }))
.sort((a, b) => b.count - a.count),
blueprintCount
};
}

// --- Grep support methods (replaces DB queries for grep handler) ---

/**
Expand Down Expand Up @@ -1484,6 +1464,7 @@ export class MemoryIndex {
totalTypes: this.typesById.size,
totalMembers: this.membersById.size,
totalAssets: this.assetsById.size,
blueprintCount: 0,
byKind: {},
byMemberKind: {},
byLanguage: {},
Expand Down Expand Up @@ -1512,6 +1493,11 @@ export class MemoryIndex {
for (const [, m] of this.membersById) {
this._stats.byMemberKind[m.memberKind] = (this._stats.byMemberKind[m.memberKind] || 0) + 1;
}

// Asset stats
for (const [, a] of this.assetsById) {
if (a.parentClass) this._stats.blueprintCount++;
}
}

// --- Phase 6: Ingest sync hooks ---
Expand Down
3 changes: 1 addition & 2 deletions src/service/query-worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@ const ALLOWED_METHODS = new Set([
'listModules',
'browseAssetFolder',
'listAssetFolders',
'getStats',
'getAssetStats'
'getStats'
]);

parentPort.on('message', (msg) => {
Expand Down
Loading