diff --git a/packages/host/app/lib/utils.ts b/packages/host/app/lib/utils.ts index 873e5c0ad73..cf0745e25e1 100644 --- a/packages/host/app/lib/utils.ts +++ b/packages/host/app/lib/utils.ts @@ -6,6 +6,10 @@ import { devSkillLocalPath, envSkillLocalPath, } from '@cardstack/runtime-common'; +export { + iconURLFor, + getRandomBackgroundURL, +} from '@cardstack/runtime-common/realm-display-defaults'; import ENV from '@cardstack/host/config/environment'; @@ -47,97 +51,6 @@ export function cleanseString(value: string) { .replace(/[^a-z0-9]$/, ''); } -export function iconURLFor(word: string) { - if (!word) { - return undefined; - } - let cleansedWord = cleanseString(word).replace(/^[0-9]+/, ''); - return iconURLs[cleansedWord.charAt(0)]; -} - -export function getRandomBackgroundURL() { - const index = Math.floor(Math.random() * backgroundURLs.length); - return backgroundURLs[index]; -} - -const iconURLs: { [letter: string]: string } = Object.freeze({ - a: 'https://boxel-images.boxel.ai/icons/Letter-a.png', - b: 'https://boxel-images.boxel.ai/icons/Letter-b.png', - c: 'https://boxel-images.boxel.ai/icons/Letter-c.png', - d: 'https://boxel-images.boxel.ai/icons/Letter-d.png', - e: 'https://boxel-images.boxel.ai/icons/Letter-e.png', - f: 'https://boxel-images.boxel.ai/icons/Letter-f.png', - g: 'https://boxel-images.boxel.ai/icons/Letter-g.png', - h: 'https://boxel-images.boxel.ai/icons/Letter-h.png', - i: 'https://boxel-images.boxel.ai/icons/Letter-i.png', - j: 'https://boxel-images.boxel.ai/icons/Letter-j.png', - k: 'https://boxel-images.boxel.ai/icons/Letter-k.png', - l: 'https://boxel-images.boxel.ai/icons/Letter-l.png', - m: 'https://boxel-images.boxel.ai/icons/Letter-m.png', - n: 'https://boxel-images.boxel.ai/icons/Letter-n.png', - o: 'https://boxel-images.boxel.ai/icons/Letter-o.png', - p: 'https://boxel-images.boxel.ai/icons/Letter-p.png', - q: 'https://boxel-images.boxel.ai/icons/Letter-q.png', - r: 'https://boxel-images.boxel.ai/icons/Letter-r.png', - s: 'https://boxel-images.boxel.ai/icons/Letter-s.png', - t: 'https://boxel-images.boxel.ai/icons/Letter-t.png', - u: 'https://boxel-images.boxel.ai/icons/Letter-u.png', - v: 'https://boxel-images.boxel.ai/icons/Letter-v.png', - w: 'https://boxel-images.boxel.ai/icons/Letter-w.png', - x: 'https://boxel-images.boxel.ai/icons/Letter-x.png', - y: 'https://boxel-images.boxel.ai/icons/Letter-y.png', - z: 'https://boxel-images.boxel.ai/icons/letter-z.png', -}); -const backgroundURLs: readonly string[] = Object.freeze([ - 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', -]); - export function urlForRealmLookup(card: CardDef) { let urlForRealmLookup = card.id ?? card[realmURL]?.href; if (!urlForRealmLookup) { diff --git a/packages/runtime-common/realm-display-defaults.ts b/packages/runtime-common/realm-display-defaults.ts new file mode 100644 index 00000000000..689fcef97df --- /dev/null +++ b/packages/runtime-common/realm-display-defaults.ts @@ -0,0 +1,108 @@ +/** + * Default icon and background URL generators for new realms. + * + * Shared by host, software-factory, and any other package that creates realms. + * Pure JS — no external dependencies. + */ + +const ICON_URLS: { [letter: string]: string } = Object.freeze({ + a: 'https://boxel-images.boxel.ai/icons/Letter-a.png', + b: 'https://boxel-images.boxel.ai/icons/Letter-b.png', + c: 'https://boxel-images.boxel.ai/icons/Letter-c.png', + d: 'https://boxel-images.boxel.ai/icons/Letter-d.png', + e: 'https://boxel-images.boxel.ai/icons/Letter-e.png', + f: 'https://boxel-images.boxel.ai/icons/Letter-f.png', + g: 'https://boxel-images.boxel.ai/icons/Letter-g.png', + h: 'https://boxel-images.boxel.ai/icons/Letter-h.png', + i: 'https://boxel-images.boxel.ai/icons/Letter-i.png', + j: 'https://boxel-images.boxel.ai/icons/Letter-j.png', + k: 'https://boxel-images.boxel.ai/icons/Letter-k.png', + l: 'https://boxel-images.boxel.ai/icons/Letter-l.png', + m: 'https://boxel-images.boxel.ai/icons/Letter-m.png', + n: 'https://boxel-images.boxel.ai/icons/Letter-n.png', + o: 'https://boxel-images.boxel.ai/icons/Letter-o.png', + p: 'https://boxel-images.boxel.ai/icons/Letter-p.png', + q: 'https://boxel-images.boxel.ai/icons/Letter-q.png', + r: 'https://boxel-images.boxel.ai/icons/Letter-r.png', + s: 'https://boxel-images.boxel.ai/icons/Letter-s.png', + t: 'https://boxel-images.boxel.ai/icons/Letter-t.png', + u: 'https://boxel-images.boxel.ai/icons/Letter-u.png', + v: 'https://boxel-images.boxel.ai/icons/Letter-v.png', + w: 'https://boxel-images.boxel.ai/icons/Letter-w.png', + x: 'https://boxel-images.boxel.ai/icons/Letter-x.png', + y: 'https://boxel-images.boxel.ai/icons/Letter-y.png', + z: 'https://boxel-images.boxel.ai/icons/letter-z.png', +}); + +const BACKGROUND_URLS: readonly string[] = Object.freeze([ + 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', +]); + +/** + * Generate a letter-based icon URL from a realm/workspace name. + * Returns undefined if the name has no alphabetic characters. + */ +export function iconURLFor(word: string): string | undefined { + if (!word) { + return undefined; + } + let cleansed = word + .toLowerCase() + .replace(/[^a-z0-9]/g, '') + .replace(/^[0-9]+/, ''); + return ICON_URLS[cleansed.charAt(0)]; +} + +/** + * Pick a random background image URL from the predefined set. + */ +export function getRandomBackgroundURL(): string { + let index = Math.floor(Math.random() * BACKGROUND_URLS.length); + return BACKGROUND_URLS[index]; +} diff --git a/packages/software-factory/README.md b/packages/software-factory/README.md index 2661103ee3d..882fe370cbd 100644 --- a/packages/software-factory/README.md +++ b/packages/software-factory/README.md @@ -18,6 +18,11 @@ the repo. Package linting currently runs `glint`, `eslint`, and `prettier`. - Docker running - Host app assets available at `http://localhost:4200/` - use `cd packages/host && pnpm serve:dist` +- Boxel icons server available at `http://localhost:4206/` + - use `cd packages/boxel-icons && pnpm serve` + - in a worktree where boxel-icons hasn't been built, symlink the dist from the main checkout: + `ln -s /path/to/boxel/packages/boxel-icons/dist packages/boxel-icons/dist` + - required before `cache:prepare` — the harness indexes cards that reference icon modules The harness starts its own seeded test Postgres, Synapse, prerender server, and isolated realm server. By default it serves the test realm and base realm from diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index 9a772200452..e9924047d31 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -12,6 +12,7 @@ "factory:agent-smoke": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/factory-agent-smoke.ts", "factory:go": "NODE_NO_WARNINGS=1 ts-node --transpileOnly src/cli/factory-entrypoint.ts", "factory:skill-smoke": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/factory-skill-smoke.ts", + "factory:tools-smoke": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/factory-tools-smoke.ts", "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"", "lint:js": "eslint . --report-unused-disable-directives --cache", diff --git a/packages/software-factory/scripts/factory-tools-smoke.ts b/packages/software-factory/scripts/factory-tools-smoke.ts new file mode 100644 index 00000000000..959f83714e4 --- /dev/null +++ b/packages/software-factory/scripts/factory-tools-smoke.ts @@ -0,0 +1,255 @@ +/** + * Smoke test for the ToolRegistry and ToolExecutor. + * + * No running services required — exercises the registry, manifest validation, + * safety checks, and a mocked realm-api round-trip entirely in-process. + * + * Usage: + * pnpm factory:tools-smoke + */ + +import type { AgentAction } from './lib/factory-agent'; +import { + ToolExecutor, + ToolNotFoundError, + ToolSafetyError, +} from './lib/factory-tool-executor'; +import { ToolRegistry } from './lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +let passed = 0; +let failed = 0; + +function check(label: string, ok: boolean, detail?: string): void { + if (ok) { + passed++; + console.log(` \u2713 ${label}`); + } else { + failed++; + console.log(` \u2717 ${label}${detail ? ` -- ${detail}` : ''}`); + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + // ----------------------------------------------------------------------- + // 1. Registry + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Tool Registry ==='); + console.log(''); + + let registry = new ToolRegistry(); + let manifests = registry.getManifests(); + + console.log(`Registered tools: ${manifests.length}`); + console.log(''); + + let byCategory: Record = {}; + for (let m of manifests) { + (byCategory[m.category] ??= []).push(m.name); + } + + for (let [category, names] of Object.entries(byCategory)) { + console.log(` ${category} (${names.length}):`); + for (let name of names) { + let manifest = registry.getManifest(name)!; + let requiredArgs = manifest.args + .filter((a) => a.required) + .map((a) => a.name); + let optionalArgs = manifest.args + .filter((a) => !a.required) + .map((a) => a.name); + console.log( + ` - ${name} [${manifest.outputFormat}]` + + (requiredArgs.length + ? ` required: ${requiredArgs.join(', ')}` + : '') + + (optionalArgs.length ? ` optional: ${optionalArgs.join(', ')}` : ''), + ); + } + console.log(''); + } + + check('has script tools', byCategory['script']?.length === 4); + check('has boxel-cli tools', byCategory['boxel-cli']?.length === 6); + check('has realm-api tools', byCategory['realm-api']?.length === 8); + check( + 'all names unique', + new Set(manifests.map((m) => m.name)).size === manifests.length, + ); + + // ----------------------------------------------------------------------- + // 2. Argument validation + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Argument Validation ==='); + console.log(''); + + let validErrors = registry.validateArgs('search-realm', { + realm: 'http://example.test/', + }); + check('valid args -> no errors', validErrors.length === 0); + + let missingErrors = registry.validateArgs('search-realm', {}); + check( + 'missing required arg -> error', + missingErrors.length > 0 && missingErrors[0].includes('realm'), + ); + + let unknownErrors = registry.validateArgs('not-a-tool', {}); + check( + 'unknown tool -> error', + unknownErrors.length > 0 && unknownErrors[0].includes('Unknown'), + ); + + // ----------------------------------------------------------------------- + // 3. Safety constraints + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Safety Constraints ==='); + console.log(''); + + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: 'https://realms.example.test/user/target/', + testRealmUrl: 'https://realms.example.test/user/target-tests/', + sourceRealmUrl: 'https://realms.example.test/user/source/', + allowedRealmPrefixes: ['https://realms.example.test/user/scratch-'], + }); + + // Unregistered tool + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'rm-rf', + } as AgentAction); + check('rejects unregistered tool', false, 'did not throw'); + } catch (err) { + check('rejects unregistered tool', err instanceof ToolNotFoundError); + } + + // Source realm targeting + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'search-realm', + toolArgs: { realm: 'https://realms.example.test/user/source/' }, + }); + check('rejects source realm', false, 'did not throw'); + } catch (err) { + check('rejects source realm', err instanceof ToolSafetyError); + } + + // Unknown realm targeting (realm-api) + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': 'https://evil.example.test/hacker/realm/', + path: 'secrets.json', + }, + }); + check('rejects unknown realm', false, 'did not throw'); + } catch (err) { + check('rejects unknown realm', err instanceof ToolSafetyError); + } + + // ----------------------------------------------------------------------- + // 4. Mocked realm-api round-trip + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Realm API Round-Trip (mock) ==='); + console.log(''); + + let mockCallCount = 0; + + let mockExecutor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: 'https://realms.example.test/user/target/', + testRealmUrl: 'https://realms.example.test/user/target-tests/', + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + mockCallCount++; + let url = String(input); + let method = init?.method ?? 'GET'; + console.log(` -> ${method} ${url}`); + return new Response( + JSON.stringify({ + data: [{ id: 'CardDef/hello', type: 'card' }], + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }) as typeof globalThis.fetch, + }); + + let readResult = await mockExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/hello.gts', + }, + }); + check('realm-read exitCode=0', readResult.exitCode === 0); + check('realm-read has output', readResult.output !== undefined); + check( + `realm-read duration ${readResult.durationMs}ms`, + readResult.durationMs >= 0, + ); + + let searchResult = await mockExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': 'https://realms.example.test/user/target/', + query: JSON.stringify({ filter: { type: { name: 'Ticket' } } }), + }, + }); + check('realm-search exitCode=0', searchResult.exitCode === 0); + + let writeResult = await mockExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-write', + toolArgs: { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/new.gts', + content: 'export class NewCard {}', + }, + }); + check('realm-write exitCode=0', writeResult.exitCode === 0); + + check(`mock fetch called ${mockCallCount} times`, mockCallCount === 3); + + // ----------------------------------------------------------------------- + // Summary + // ----------------------------------------------------------------------- + + console.log(''); + console.log('==========================='); + console.log(` ${passed} passed, ${failed} failed`); + console.log('==========================='); + console.log(''); + + if (failed > 0) { + process.exit(1); + } +} + +main().catch((err: unknown) => { + console.error( + 'Smoke test failed:', + err instanceof Error ? err.message : String(err), + ); + process.exit(1); +}); diff --git a/packages/software-factory/scripts/lib/factory-tool-executor.ts b/packages/software-factory/scripts/lib/factory-tool-executor.ts new file mode 100644 index 00000000000..9bfa9087979 --- /dev/null +++ b/packages/software-factory/scripts/lib/factory-tool-executor.ts @@ -0,0 +1,926 @@ +import { spawn } from 'node:child_process'; +import { resolve } from 'node:path'; + +import { + iconURLFor, + getRandomBackgroundURL, +} from '@cardstack/runtime-common/realm-display-defaults'; + +import type { AgentAction, ToolResult } from './factory-agent'; +import type { ToolRegistry } from './factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const DEFAULT_TIMEOUT_MS = 60_000; + +/** + * Map from script tool name to the script file that implements it. + * Paths are relative to `packages/software-factory/scripts/`. + */ +const SCRIPT_FILE_MAP: Record = { + 'search-realm': 'boxel-search.ts', + 'pick-ticket': 'pick-ticket.ts', + 'get-session': 'boxel-session.ts', + 'run-realm-tests': 'run-realm-tests.ts', +}; + +/** + * Map from boxel-cli tool name to the `npx boxel` subcommand. + */ +const BOXEL_CLI_COMMAND_MAP: Record = { + 'boxel-sync': 'sync', + 'boxel-push': 'push', + 'boxel-pull': 'pull', + 'boxel-status': 'status', + 'boxel-create': 'create', + 'boxel-history': 'history', +}; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ToolExecutorConfig { + /** Absolute path to the software-factory package root. */ + packageRoot: string; + /** Target realm URL — tools may only target this realm. */ + targetRealmUrl: string; + /** Test realm URL — tools may also target this realm. */ + testRealmUrl: string; + /** Additional scratch realm URL prefixes that are allowed. */ + allowedRealmPrefixes?: string[]; + /** Source realm URL — tools must NEVER target this realm. */ + sourceRealmUrl?: string; + /** Fetch implementation for realm API calls. */ + fetch?: typeof globalThis.fetch; + /** Authorization header value for realm API calls. */ + authorization?: string; + /** Per-invocation timeout in ms (default: 60 000). */ + timeoutMs?: number; + /** Optional log function for auditability. */ + log?: (entry: ToolExecutionLogEntry) => void; + /** Matrix homeserver URL (for post-create account data update). */ + matrixUrl?: string; + /** Matrix access token (from Matrix login). */ + matrixAccessToken?: string; + /** Matrix user ID (e.g. @factory:localhost). */ + matrixUserId?: string; +} + +export interface ToolExecutionLogEntry { + tool: string; + category: 'script' | 'boxel-cli' | 'realm-api'; + args: Record; + exitCode: number; + durationMs: number; + error?: string; +} + +// --------------------------------------------------------------------------- +// Error classes +// --------------------------------------------------------------------------- + +export class ToolSafetyError extends Error { + constructor(message: string) { + super(message); + this.name = 'ToolSafetyError'; + } +} + +export class ToolTimeoutError extends Error { + constructor(tool: string, timeoutMs: number) { + super(`Tool "${tool}" timed out after ${timeoutMs}ms`); + this.name = 'ToolTimeoutError'; + } +} + +export class ToolNotFoundError extends Error { + constructor(tool: string) { + super(`Unregistered tool: "${tool}"`); + this.name = 'ToolNotFoundError'; + } +} + +// --------------------------------------------------------------------------- +// ToolExecutor +// --------------------------------------------------------------------------- + +export class ToolExecutor { + private registry: ToolRegistry; + private config: ToolExecutorConfig; + private timeoutMs: number; + + constructor(registry: ToolRegistry, config: ToolExecutorConfig) { + this.registry = registry; + this.config = config; + this.timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; + } + + /** + * Execute a validated `invoke_tool` action and return the result. + * + * The executor: + * 1. Validates the tool name against the registry + * 2. Validates arguments against the manifest + * 3. Enforces safety constraints (no source realm targeting) + * 4. Dispatches to the appropriate sub-executor + * 5. Captures output as a ToolResult + */ + async execute(action: AgentAction): Promise { + let toolName = action.tool; + if (!toolName) { + throw new ToolNotFoundError('(empty)'); + } + + let manifest = this.registry.getManifest(toolName); + if (!manifest) { + throw new ToolNotFoundError(toolName); + } + + let toolArgs = action.toolArgs ?? {}; + + // Validate required args + let argErrors = this.registry.validateArgs(toolName, toolArgs); + if (argErrors.length > 0) { + throw new Error( + `Invalid arguments for tool "${toolName}": ${argErrors.join('; ')}`, + ); + } + + // Safety: reject source realm targeting + this.enforceRealmSafety(toolName, toolArgs); + + let start = Date.now(); + let result: ToolResult; + + try { + switch (manifest.category) { + case 'script': + result = await this.executeScript(toolName, toolArgs); + break; + case 'boxel-cli': + result = await this.executeBoxelCli(toolName, toolArgs); + break; + case 'realm-api': + result = await this.executeRealmApi(toolName, toolArgs); + break; + default: + throw new Error(`Unknown tool category: ${manifest.category}`); + } + } catch (error) { + let durationMs = Date.now() - start; + let errorMessage = error instanceof Error ? error.message : String(error); + + this.logExecution({ + tool: toolName, + category: manifest.category, + args: toolArgs, + exitCode: 1, + durationMs, + error: errorMessage, + }); + + // Re-throw safety and timeout errors as-is + if ( + error instanceof ToolSafetyError || + error instanceof ToolTimeoutError || + error instanceof ToolNotFoundError + ) { + throw error; + } + + return { + tool: toolName, + exitCode: 1, + output: { error: errorMessage }, + durationMs, + }; + } + + this.logExecution({ + tool: toolName, + category: manifest.category, + args: toolArgs, + exitCode: result.exitCode, + durationMs: result.durationMs, + }); + + return result; + } + + // ------------------------------------------------------------------------- + // Safety + // ------------------------------------------------------------------------- + + private enforceRealmSafety( + toolName: string, + toolArgs: Record, + ): void { + // Source realm protection (when configured) + let sourceUrl = this.config.sourceRealmUrl; + if (sourceUrl) { + let normalizedSource = ensureTrailingSlash(sourceUrl); + + let realmArgNames = [ + 'realm', + 'realm-url', + 'realm-server-url', + 'local-dir', + 'path', + ]; + + for (let argName of realmArgNames) { + let value = toolArgs[argName]; + if (typeof value === 'string' && looksLikeUrl(value)) { + let normalizedValue = ensureTrailingSlash(value); + if (normalizedValue === normalizedSource) { + throw new ToolSafetyError( + `Tool "${toolName}" cannot target the source realm: ${sourceUrl}`, + ); + } + } + } + } + + // Allowed-target validation for all URL args across all tool categories. + // Prevents SSRF and token exfiltration via the propagated Authorization header. + let urlArgNames = ['realm', 'realm-url']; + for (let argName of urlArgNames) { + let value = toolArgs[argName]; + if (typeof value === 'string' && looksLikeUrl(value)) { + this.validateRealmTarget(toolName, value); + } + } + + // realm-server-url must match one of the allowed realm origins + let serverUrl = toolArgs['realm-server-url']; + if (typeof serverUrl === 'string' && looksLikeUrl(serverUrl)) { + this.validateRealmServerTarget(toolName, serverUrl); + } + + // Extra validation for destructive operations + this.validateDestructiveOps(toolName, toolArgs); + } + + private validateRealmTarget(toolName: string, realmUrl: string): void { + let normalized = ensureTrailingSlash(realmUrl); + let target = ensureTrailingSlash(this.config.targetRealmUrl); + let test = ensureTrailingSlash(this.config.testRealmUrl); + + // Exact realm matches (with trailing slash normalization) + let exactAllowed = [target, test]; + + // Prefix matches (no trailing slash — these are URL path prefixes) + let prefixAllowed = this.config.allowedRealmPrefixes ?? []; + + let isAllowed = + exactAllowed.some((exact) => normalized === exact) || + prefixAllowed.some((prefix) => normalized.startsWith(prefix)); + + if (!isAllowed) { + throw new ToolSafetyError( + `Tool "${toolName}" targets realm "${realmUrl}" which is not in the allowed list. ` + + `Allowed: ${[...exactAllowed, ...prefixAllowed].join(', ')}`, + ); + } + } + + /** + * Validate that a realm-server-url arg points to a server that hosts + * one of the allowed realms (origin match against target/test/prefixes). + */ + private validateRealmServerTarget(toolName: string, serverUrl: string): void { + let normalizedServer: string; + try { + normalizedServer = new URL(serverUrl).origin; + } catch { + throw new ToolSafetyError( + `Tool "${toolName}" has invalid realm-server-url: "${serverUrl}"`, + ); + } + + let allowedOrigins = new Set(); + try { + allowedOrigins.add(new URL(this.config.targetRealmUrl).origin); + } catch { + // skip invalid + } + try { + allowedOrigins.add(new URL(this.config.testRealmUrl).origin); + } catch { + // skip invalid + } + for (let prefix of this.config.allowedRealmPrefixes ?? []) { + try { + allowedOrigins.add(new URL(prefix).origin); + } catch { + // skip invalid + } + } + + if (!allowedOrigins.has(normalizedServer)) { + throw new ToolSafetyError( + `Tool "${toolName}" targets server "${serverUrl}" which is not in the allowed origins. ` + + `Allowed: ${[...allowedOrigins].join(', ')}`, + ); + } + } + + private validateDestructiveOps( + toolName: string, + toolArgs: Record, + ): void { + // realm-delete and realm-atomic with remove ops need extra care + if (toolName === 'realm-delete') { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string') { + this.validateRealmTarget(toolName, realmUrl); + } + } + + if (toolName === 'realm-atomic') { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string') { + this.validateRealmTarget(toolName, realmUrl); + } + } + + // boxel-push with --delete + if (toolName === 'boxel-push' && toolArgs['delete']) { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string' && looksLikeUrl(realmUrl)) { + this.validateRealmTarget(toolName, realmUrl); + } + } + + // realm-create is allowed but logged — the orchestrator trusts the agent + // chose it deliberately within the allowed realm set. + } + + // ------------------------------------------------------------------------- + // Sub-executors + // ------------------------------------------------------------------------- + + private async executeScript( + toolName: string, + toolArgs: Record, + ): Promise { + let scriptFile = SCRIPT_FILE_MAP[toolName]; + if (!scriptFile) { + throw new Error(`No script file mapped for tool "${toolName}"`); + } + + let scriptPath = resolve(this.config.packageRoot, 'scripts', scriptFile); + + let cliArgs = buildCliArgs(toolArgs); + + return this.spawnProcess( + toolName, + 'npx', + ['ts-node', '--transpileOnly', scriptPath, ...cliArgs], + 'json', + ); + } + + private async executeBoxelCli( + toolName: string, + toolArgs: Record, + ): Promise { + let subcommand = BOXEL_CLI_COMMAND_MAP[toolName]; + if (!subcommand) { + throw new Error(`No boxel-cli command mapped for tool "${toolName}"`); + } + + let cliArgs = buildBoxelCliArgs(toolName, subcommand, toolArgs); + + return this.spawnProcess(toolName, 'npx', ['boxel', ...cliArgs], 'text'); + } + + private async executeRealmApi( + toolName: string, + toolArgs: Record, + ): Promise { + let fetchImpl = this.config.fetch ?? globalThis.fetch; + let start = Date.now(); + + let { url, method, headers, body } = buildRealmApiRequest( + toolName, + toolArgs, + this.config.authorization, + ); + + let controller = new AbortController(); + let timeout = setTimeout(() => controller.abort(), this.timeoutMs); + + try { + let response = await fetchImpl(url, { + method, + headers, + body, + signal: controller.signal, + }); + + let durationMs = Date.now() - start; + let responseBody: unknown; + + let contentType = response.headers.get('content-type') ?? ''; + let rawText = await response.text(); + if (contentType.includes('json') && rawText.length > 0) { + try { + responseBody = JSON.parse(rawText); + } catch { + responseBody = rawText; + } + } else { + responseBody = rawText; + } + + // Some endpoints return important values in headers (e.g. _server-session + // returns the JWT in the Authorization header with an empty/null body). + let authorizationHeader = response.headers.get('authorization'); + if (toolName === 'realm-server-session' && authorizationHeader) { + responseBody = { token: authorizationHeader }; + } + + // After successful realm-create, update Matrix account data to include + // the new realm in the user's realm list (matching host app behavior). + if (toolName === 'realm-create' && response.ok) { + let createdRealmUrl = extractCreatedRealmUrl(responseBody); + if (createdRealmUrl) { + await this.appendRealmToMatrixAccountData(fetchImpl, createdRealmUrl); + } + } + + return { + tool: toolName, + exitCode: response.ok ? 0 : 1, + output: response.ok + ? responseBody + : { + error: `HTTP ${response.status}`, + body: responseBody, + }, + durationMs, + }; + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') { + throw new ToolTimeoutError(toolName, this.timeoutMs); + } + throw error; + } finally { + clearTimeout(timeout); + } + } + + // ------------------------------------------------------------------------- + // Process spawning + // ------------------------------------------------------------------------- + + private spawnProcess( + toolName: string, + command: string, + args: string[], + outputFormat: 'json' | 'text', + ): Promise { + return new Promise((resolvePromise, reject) => { + let start = Date.now(); + let stdout = ''; + let stderr = ''; + + let child = spawn(command, args, { + cwd: this.config.packageRoot, + env: { ...process.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let timer = setTimeout(() => { + child.kill('SIGTERM'); + // Give the process a moment to clean up, then force kill + setTimeout(() => { + if (!child.killed) { + child.kill('SIGKILL'); + } + }, 5000); + reject(new ToolTimeoutError(toolName, this.timeoutMs)); + }, this.timeoutMs); + + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + + child.on('error', (error: Error) => { + clearTimeout(timer); + reject(error); + }); + + child.on('close', (code: number | null) => { + clearTimeout(timer); + let durationMs = Date.now() - start; + let exitCode = code ?? 1; + + let output: unknown; + if (outputFormat === 'json') { + try { + output = JSON.parse(stdout.trim()); + } catch { + output = { + raw: stdout.trim(), + ...(stderr.trim() ? { stderr: stderr.trim() } : {}), + }; + } + } else { + output = stdout.trim(); + if (stderr.trim() && exitCode !== 0) { + output = `${stdout.trim()}\n\nSTDERR:\n${stderr.trim()}`; + } + } + + resolvePromise({ + tool: toolName, + exitCode, + output, + durationMs, + }); + }); + }); + } + + // ------------------------------------------------------------------------- + // Logging + // ------------------------------------------------------------------------- + + private logExecution(entry: ToolExecutionLogEntry): void { + this.config.log?.(entry); + } + + // ------------------------------------------------------------------------- + // Matrix account data + // ------------------------------------------------------------------------- + + /** + * After realm creation, append the new realm URL to the user's Matrix + * account data so the realm appears in their realm list. Matches the host + * app's `appendRealmToAccountData` behavior (matrix-service.ts:639-653). + * + * Silently skips when Matrix config is not provided. + */ + private async appendRealmToMatrixAccountData( + fetchImpl: typeof globalThis.fetch, + newRealmUrl: string, + ): Promise { + let { matrixUrl, matrixAccessToken, matrixUserId } = this.config; + if (!matrixUrl || !matrixAccessToken || !matrixUserId) { + return; + } + + let baseUrl = ensureTrailingSlash(matrixUrl); + let encodedUserId = encodeURIComponent(matrixUserId); + let accountDataUrl = `${baseUrl}_matrix/client/v3/user/${encodedUserId}/account_data/app.boxel.realms`; + let authHeaders = { + Authorization: `Bearer ${matrixAccessToken}`, + 'Content-Type': 'application/json', + }; + + // GET current realm list + let existingRealms: string[] = []; + try { + let getResponse = await fetchImpl(accountDataUrl, { + method: 'GET', + headers: authHeaders, + }); + if (getResponse.ok) { + let data = (await getResponse.json()) as { realms?: string[] }; + existingRealms = data.realms ?? []; + } + // 404 means no account data yet — start with empty list + } catch { + // Network error reading account data — proceed with empty list + } + + // Append and PUT + let updatedRealms = [...existingRealms, newRealmUrl]; + try { + await fetchImpl(accountDataUrl, { + method: 'PUT', + headers: authHeaders, + body: JSON.stringify({ realms: updatedRealms }), + }); + } catch { + // Best-effort — don't fail the realm-create if account data update fails + } + } +} + +// --------------------------------------------------------------------------- +// Helpers: CLI arg building +// --------------------------------------------------------------------------- + +function buildCliArgs(toolArgs: Record): string[] { + let args: string[] = []; + + for (let [key, value] of Object.entries(toolArgs)) { + if (value === undefined || value === null) { + continue; + } + + if (typeof value === 'boolean') { + if (value) { + args.push(`--${key}`); + } + } else if (Array.isArray(value)) { + for (let item of value) { + args.push(`--${key}`, String(item)); + } + } else { + args.push(`--${key}`, String(value)); + } + } + + return args; +} + +function buildBoxelCliArgs( + _toolName: string, + subcommand: string, + toolArgs: Record, +): string[] { + let args: string[] = [subcommand]; + + // Certain tools use positional args rather than flags + switch (subcommand) { + case 'sync': { + let path = toolArgs['path']; + if (typeof path === 'string') { + args.push(path); + } + if (typeof toolArgs['prefer'] === 'string') { + args.push(`--prefer-${toolArgs['prefer']}`); + } + if (toolArgs['dry-run']) { + args.push('--dry-run'); + } + break; + } + case 'push': { + let localDir = toolArgs['local-dir']; + let realmUrl = toolArgs['realm-url']; + if (typeof localDir === 'string') { + args.push(localDir); + } + if (typeof realmUrl === 'string') { + args.push(realmUrl); + } + if (toolArgs['delete']) { + args.push('--delete'); + } + if (toolArgs['dry-run']) { + args.push('--dry-run'); + } + break; + } + case 'pull': { + let realmUrl = toolArgs['realm-url']; + let localDir = toolArgs['local-dir']; + if (typeof realmUrl === 'string') { + args.push(realmUrl); + } + if (typeof localDir === 'string') { + args.push(localDir); + } + if (toolArgs['delete']) { + args.push('--delete'); + } + if (toolArgs['dry-run']) { + args.push('--dry-run'); + } + break; + } + case 'status': { + let path = toolArgs['path']; + if (typeof path === 'string') { + args.push(path); + } + if (toolArgs['all']) { + args.push('--all'); + } + if (toolArgs['pull']) { + args.push('--pull'); + } + break; + } + case 'create': { + let endpoint = toolArgs['endpoint']; + let name = toolArgs['name']; + if (typeof endpoint === 'string') { + args.push(endpoint); + } + if (typeof name === 'string') { + args.push(name); + } + break; + } + case 'history': { + let path = toolArgs['path']; + if (typeof path === 'string') { + args.push(path); + } + if (typeof toolArgs['message'] === 'string') { + args.push('-m', toolArgs['message']); + } + break; + } + default: + // Fall through to generic flag building + args.push(...buildCliArgs(toolArgs)); + } + + return args; +} + +// --------------------------------------------------------------------------- +// Helpers: Realm API request building +// --------------------------------------------------------------------------- + +interface RealmApiRequestParams { + url: string; + method: string; + headers: Record; + body?: string; +} + +function buildRealmApiRequest( + toolName: string, + toolArgs: Record, + authorization?: string, +): RealmApiRequestParams { + let headers: Record = { + Accept: 'application/json', + }; + + if (authorization) { + headers['Authorization'] = authorization; + } + + switch (toolName) { + case 'realm-read': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let path = String(toolArgs['path']); + let accept = + typeof toolArgs['accept'] === 'string' + ? toolArgs['accept'] + : 'application/vnd.card+source'; + return { + url: `${realmUrl}${path}`, + method: 'GET', + headers: { ...headers, Accept: accept }, + }; + } + + case 'realm-write': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let path = String(toolArgs['path']); + let content = String(toolArgs['content']); + let contentType = + typeof toolArgs['content-type'] === 'string' + ? toolArgs['content-type'] + : 'application/vnd.card+source'; + return { + url: `${realmUrl}${path}`, + method: 'POST', + headers: { ...headers, 'Content-Type': contentType }, + body: content, + }; + } + + case 'realm-delete': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let path = String(toolArgs['path']); + return { + url: `${realmUrl}${path}`, + method: 'DELETE', + headers, + }; + } + + case 'realm-atomic': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let operations = String(toolArgs['operations']); + return { + url: `${realmUrl}_atomic`, + method: 'POST', + headers: { + ...headers, + 'Content-Type': 'application/vnd.api+json', + }, + body: JSON.stringify({ 'atomic:operations': JSON.parse(operations) }), + }; + } + + case 'realm-search': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let query = String(toolArgs['query']); + return { + url: `${realmUrl}_search`, + method: 'QUERY', + headers: { + ...headers, + Accept: 'application/vnd.card+json', + 'Content-Type': 'application/json', + }, + body: query, + }; + } + + case 'realm-create': { + let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); + let name = String(toolArgs['name']); + let endpoint = String(toolArgs['endpoint']); + let iconURL = + typeof toolArgs['iconURL'] === 'string' + ? toolArgs['iconURL'] + : iconURLFor(name); + let backgroundURL = + typeof toolArgs['backgroundURL'] === 'string' + ? toolArgs['backgroundURL'] + : getRandomBackgroundURL(); + return { + url: `${serverUrl}_create-realm`, + method: 'POST', + headers: { + ...headers, + Accept: 'application/vnd.api+json', + 'Content-Type': 'application/vnd.api+json', + }, + body: JSON.stringify({ + data: { + type: 'realm', + attributes: { + name, + endpoint, + ...(iconURL ? { iconURL } : {}), + ...(backgroundURL ? { backgroundURL } : {}), + }, + }, + }), + }; + } + + case 'realm-server-session': { + let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); + let openidToken = String(toolArgs['openid-token']); + return { + url: `${serverUrl}_server-session`, + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + body: JSON.stringify({ access_token: openidToken }), + }; + } + + case 'realm-auth': { + let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); + return { + url: `${serverUrl}_realm-auth`, + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + }; + } + + default: + throw new Error(`Unknown realm-api tool: "${toolName}"`); + } +} + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- + +function ensureTrailingSlash(url: string): string { + return url.endsWith('/') ? url : `${url}/`; +} + +function looksLikeUrl(value: string): boolean { + return value.startsWith('http://') || value.startsWith('https://'); +} + +function extractCreatedRealmUrl(responseBody: unknown): string | undefined { + if ( + typeof responseBody === 'object' && + responseBody !== null && + 'data' in responseBody + ) { + let data = (responseBody as { data: unknown }).data; + if (typeof data === 'object' && data !== null && 'id' in data) { + let id = (data as { id: unknown }).id; + if (typeof id === 'string') { + return id; + } + } + } + return undefined; +} diff --git a/packages/software-factory/scripts/lib/factory-tool-registry.ts b/packages/software-factory/scripts/lib/factory-tool-registry.ts new file mode 100644 index 00000000000..a2bd31b9d0c --- /dev/null +++ b/packages/software-factory/scripts/lib/factory-tool-registry.ts @@ -0,0 +1,626 @@ +import type { ToolArg, ToolManifest } from './factory-agent'; + +// --------------------------------------------------------------------------- +// Built-in tool manifests +// --------------------------------------------------------------------------- + +const SCRIPT_TOOLS: ToolManifest[] = [ + { + name: 'search-realm', + description: + 'Search for cards in a realm by type, field values, and sort criteria.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm', + type: 'string', + required: true, + description: 'Target realm URL', + }, + { + name: 'type-name', + type: 'string', + required: false, + description: 'Filter by card type name', + }, + { + name: 'type-module', + type: 'string', + required: false, + description: 'Filter by card type module', + }, + { + name: 'eq', + type: 'string', + required: false, + description: 'Equality filter as "field=value" (repeatable)', + }, + { + name: 'contains', + type: 'string', + required: false, + description: 'Contains filter as "field=value" (repeatable)', + }, + { + name: 'sort', + type: 'string', + required: false, + description: 'Sort as "field:direction" (repeatable)', + }, + { + name: 'size', + type: 'number', + required: false, + description: 'Page size', + }, + { + name: 'page', + type: 'number', + required: false, + description: 'Page number', + }, + ], + }, + { + name: 'pick-ticket', + description: 'Find tickets by status, priority, project, or agent.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm', + type: 'string', + required: true, + description: 'Target realm URL', + }, + { + name: 'status', + type: 'string', + required: false, + description: + 'Comma-separated status filter (default: backlog,in_progress,review)', + }, + { + name: 'project', + type: 'string', + required: false, + description: 'Filter by project ID', + }, + { + name: 'agent', + type: 'string', + required: false, + description: 'Filter by assigned agent ID', + }, + { + name: 'module', + type: 'string', + required: false, + description: 'Ticket schema module URL', + }, + ], + }, + { + name: 'get-session', + description: + 'Generate authenticated browser session tokens for realm access.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm', + type: 'string', + required: false, + description: 'Specific realm URL to include (repeatable)', + }, + ], + }, + { + name: 'run-realm-tests', + description: + 'Execute Playwright tests in an isolated scratch realm with fixture setup and teardown.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm-path', + type: 'string', + required: false, + description: 'Source realm directory', + }, + { + name: 'realm-url', + type: 'string', + required: false, + description: 'Source realm URL', + }, + { + name: 'spec-dir', + type: 'string', + required: false, + description: 'Test directory (default: tests)', + }, + { + name: 'fixtures-dir', + type: 'string', + required: false, + description: 'Fixtures directory (default: tests/fixtures)', + }, + { + name: 'endpoint', + type: 'string', + required: false, + description: 'Realm endpoint name', + }, + { + name: 'scratch-root', + type: 'string', + required: false, + description: 'Base dir for test realms', + }, + ], + }, +]; + +const BOXEL_CLI_TOOLS: ToolManifest[] = [ + { + name: 'boxel-sync', + description: 'Bidirectional sync between local workspace and realm server.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'path', + type: 'string', + required: true, + description: 'Local workspace path', + }, + { + name: 'prefer', + type: 'string', + required: false, + description: 'Conflict strategy: "local", "remote", or "newest"', + }, + { + name: 'dry-run', + type: 'boolean', + required: false, + description: 'Preview only, no changes', + }, + ], + }, + { + name: 'boxel-push', + description: 'One-way upload from local directory to realm.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'local-dir', + type: 'string', + required: true, + description: 'Local directory path', + }, + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Target realm URL', + }, + { + name: 'delete', + type: 'boolean', + required: false, + description: 'Remove orphaned remote files', + }, + { + name: 'dry-run', + type: 'boolean', + required: false, + description: 'Preview only, no changes', + }, + ], + }, + { + name: 'boxel-pull', + description: 'One-way download from realm to local directory.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Source realm URL', + }, + { + name: 'local-dir', + type: 'string', + required: true, + description: 'Local directory path', + }, + { + name: 'delete', + type: 'boolean', + required: false, + description: 'Delete local files not on remote', + }, + { + name: 'dry-run', + type: 'boolean', + required: false, + description: 'Preview only, no changes', + }, + ], + }, + { + name: 'boxel-status', + description: 'Check sync status of a workspace.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'path', + type: 'string', + required: true, + description: 'Local workspace path', + }, + { + name: 'all', + type: 'boolean', + required: false, + description: 'Check all workspaces', + }, + { + name: 'pull', + type: 'boolean', + required: false, + description: 'Auto-pull remote changes', + }, + ], + }, + { + name: 'boxel-create', + description: 'Create a new workspace/realm endpoint.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'endpoint', + type: 'string', + required: true, + description: 'Endpoint type', + }, + { + name: 'name', + type: 'string', + required: true, + description: 'Workspace name', + }, + ], + }, + { + name: 'boxel-history', + description: 'View or create checkpoints for a workspace.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'path', + type: 'string', + required: true, + description: 'Local workspace path', + }, + { + name: 'message', + type: 'string', + required: false, + description: 'Checkpoint message', + }, + ], + }, +]; + +const REALM_API_TOOLS: ToolManifest[] = [ + { + name: 'realm-read', + description: 'Fetch a card or file from a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'path', + type: 'string', + required: true, + description: 'Card or file path within the realm', + }, + { + name: 'accept', + type: 'string', + required: false, + description: + 'Accept header. Default: application/vnd.card+source (raw source — path MUST include file extension, e.g. CardDef/my-card.gts). ' + + 'Use application/vnd.card+json for computed card instances with resolved fields (path must NOT include extension, e.g. Card/instance).', + }, + ], + }, + { + name: 'realm-write', + description: 'Create or update a card or file in a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'path', + type: 'string', + required: true, + description: 'Card or file path within the realm', + }, + { + name: 'content', + type: 'string', + required: true, + description: 'File content to write', + }, + { + name: 'content-type', + type: 'string', + required: false, + description: + 'Content-Type header (default: application/vnd.card+source)', + }, + ], + }, + { + name: 'realm-delete', + description: 'Delete a card or file from a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'path', + type: 'string', + required: true, + description: 'Card or file path to delete', + }, + ], + }, + { + name: 'realm-atomic', + description: 'Batch operations that succeed or fail atomically.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'operations', + type: 'string', + required: true, + description: + 'JSON array of operations: [{"op":"add|update|remove","href":"...","data":{...}}]', + }, + ], + }, + { + name: 'realm-search', + description: 'Search for cards using structured queries.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'query', + type: 'string', + required: true, + description: 'JSON search query object', + }, + ], + }, + { + name: 'realm-create', + description: 'Create a new realm on the realm server.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-server-url', + type: 'string', + required: true, + description: 'Realm server base URL', + }, + { + name: 'name', + type: 'string', + required: true, + description: 'Display name for the new realm', + }, + { + name: 'endpoint', + type: 'string', + required: true, + description: + 'URL path segment for the new realm (e.g. "user/my-realm")', + }, + { + name: 'iconURL', + type: 'string', + required: false, + description: + 'Icon URL for the realm. Defaults to a letter-based icon derived from the realm name.', + }, + { + name: 'backgroundURL', + type: 'string', + required: false, + description: + 'Background image URL for the realm. Defaults to a random background image.', + }, + ], + }, + { + name: 'realm-server-session', + description: + 'Obtain a realm server JWT for management operations. Returns the JWT in the output.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-server-url', + type: 'string', + required: true, + description: 'Realm server base URL', + }, + { + name: 'openid-token', + type: 'string', + required: true, + description: + 'OpenID access_token obtained from the Matrix server via /openid/request_token', + }, + ], + }, + { + name: 'realm-auth', + description: + 'Get per-realm JWTs for all realms accessible to the authenticated user.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-server-url', + type: 'string', + required: true, + description: 'Realm server base URL', + }, + ], + }, +]; + +// --------------------------------------------------------------------------- +// ToolRegistry +// --------------------------------------------------------------------------- + +export class ToolRegistry { + private manifestsByName: Map; + + constructor(manifests?: ToolManifest[]) { + let allManifests = manifests ?? [ + ...SCRIPT_TOOLS, + ...BOXEL_CLI_TOOLS, + ...REALM_API_TOOLS, + ]; + let map = new Map(); + for (let manifest of allManifests) { + if (map.has(manifest.name)) { + throw new Error( + `Duplicate tool manifest name "${manifest.name}" detected in ToolRegistry`, + ); + } + map.set(manifest.name, manifest); + } + this.manifestsByName = map; + } + + /** Return all registered tool manifests. */ + getManifests(): ToolManifest[] { + return [...this.manifestsByName.values()]; + } + + /** Look up a single manifest by tool name. Returns undefined if not found. */ + getManifest(name: string): ToolManifest | undefined { + return this.manifestsByName.get(name); + } + + /** Check if a tool name is registered. */ + has(name: string): boolean { + return this.manifestsByName.has(name); + } + + /** Number of registered tools. */ + get size(): number { + return this.manifestsByName.size; + } + + /** + * Validate that a tool invocation's arguments satisfy the manifest. + * Returns an array of error messages (empty if valid). + */ + validateArgs( + toolName: string, + toolArgs: Record | undefined, + ): string[] { + let manifest = this.manifestsByName.get(toolName); + if (!manifest) { + return [`Unknown tool: "${toolName}"`]; + } + + let errors: string[] = []; + let requiredArgs = manifest.args.filter((a) => a.required); + + for (let arg of requiredArgs) { + let value = toolArgs?.[arg.name]; + let isEmpty = + value === undefined || + value === null || + (typeof value === 'string' && value.trim() === ''); + if (isEmpty) { + errors.push( + `Missing required argument "${arg.name}" for tool "${toolName}"`, + ); + } + } + + return errors; + } +} + +// --------------------------------------------------------------------------- +// Convenience: default registry singleton +// --------------------------------------------------------------------------- + +let _defaultRegistry: ToolRegistry | undefined; + +export function getDefaultToolRegistry(): ToolRegistry { + if (!_defaultRegistry) { + _defaultRegistry = new ToolRegistry(); + } + return _defaultRegistry; +} + +// Re-export the built-in manifest arrays for testing +export { + SCRIPT_TOOLS, + BOXEL_CLI_TOOLS, + REALM_API_TOOLS, + type ToolArg, + type ToolManifest, +}; diff --git a/packages/software-factory/src/factory-target-realm.ts b/packages/software-factory/src/factory-target-realm.ts index 31a57837ccb..ba6480d60f2 100644 --- a/packages/software-factory/src/factory-target-realm.ts +++ b/packages/software-factory/src/factory-target-realm.ts @@ -1,6 +1,10 @@ import { getMatrixUsername } from '@cardstack/runtime-common/matrix-client'; import { APP_BOXEL_REALMS_EVENT_TYPE } from '@cardstack/runtime-common/matrix-constants'; import { ensureTrailingSlash } from '@cardstack/runtime-common/paths'; +import { + iconURLFor, + getRandomBackgroundURL, +} from '@cardstack/runtime-common/realm-display-defaults'; import { SupportedMimeType } from '@cardstack/runtime-common/router'; import { @@ -105,8 +109,8 @@ async function createRealm( attributes: { endpoint, name: endpoint, - iconURL: iconURLForRealmName(endpoint), - backgroundURL: randomBackgroundURL(), + iconURL: iconURLFor(endpoint), + backgroundURL: getRandomBackgroundURL(), }, }, }), @@ -377,70 +381,3 @@ function normalizeOptionalString( let trimmed = value.trim(); return trimmed === '' ? undefined : trimmed; } - -// Mirrors iconURLFor() from packages/host/app/lib/utils.ts -function iconURLForRealmName(name: string): string { - let letter = name - .toLowerCase() - .replace(/[^a-z]/g, '') - .charAt(0); - if (!letter) { - letter = 'b'; // fallback for names starting with numbers/symbols - } - return `https://boxel-images.boxel.ai/icons/Letter-${letter}.png`; -} - -// Mirrors getRandomBackgroundURL() from packages/host/app/lib/utils.ts -const backgroundURLs = [ - 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', -] as const; - -function randomBackgroundURL(): string { - return backgroundURLs[Math.floor(Math.random() * backgroundURLs.length)]; -} diff --git a/packages/software-factory/tests/factory-tool-executor.integration.test.ts b/packages/software-factory/tests/factory-tool-executor.integration.test.ts new file mode 100644 index 00000000000..121edab89e4 --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.integration.test.ts @@ -0,0 +1,644 @@ +import { createServer, type IncomingMessage, type Server } from 'node:http'; +import { module, test } from 'qunit'; + +import { ToolExecutor } from '../scripts/lib/factory-tool-executor'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Test server helpers +// --------------------------------------------------------------------------- + +interface CapturedRequest { + method: string; + url: string; + headers: Record; + body: string; +} + +function startTestServer( + handler: ( + req: CapturedRequest, + respond: ( + status: number, + body: unknown, + headers?: Record, + ) => void, + ) => void, +): Promise<{ server: Server; origin: string }> { + return new Promise((resolve) => { + let server = createServer(async (req: IncomingMessage, res) => { + let body = ''; + for await (let chunk of req) { + body += chunk; + } + + let captured: CapturedRequest = { + method: req.method ?? 'GET', + url: req.url ?? '/', + headers: req.headers, + body, + }; + + handler(captured, (status, responseBody, headers) => { + res.writeHead(status, { + 'Content-Type': 'application/json', + ...headers, + }); + res.end( + responseBody !== null && responseBody !== undefined + ? JSON.stringify(responseBody) + : '', + ); + }); + }); + + server.listen(0, () => { + let address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected test server to bind to a TCP port'); + } + let origin = `http://127.0.0.1:${address.port}`; + resolve({ server, origin }); + }); + }); +} + +function stopServer(server: Server): Promise { + return new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())), + ); +} + +// --------------------------------------------------------------------------- +// Integration tests: realm-api tools against a real HTTP server +// --------------------------------------------------------------------------- + +module('factory-tool-executor integration > realm-api requests', function () { + test('realm-read sends correct GET with Authorization and Accept headers', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { data: { id: 'Card/hello', type: 'card' } }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': realmUrl, + path: 'Card/hello.gts', + }, + }); + + assert.strictEqual(result.exitCode, 0, 'exitCode is 0'); + assert.strictEqual(captured!.method, 'GET'); + assert.strictEqual(captured!.url, '/user/target/Card/hello.gts'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + assert.strictEqual( + captured!.headers.accept, + 'application/vnd.card+source', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-write sends correct POST with content and headers', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { ok: true }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-write', + toolArgs: { + 'realm-url': realmUrl, + path: 'CardDef/my-card.gts', + content: 'export class MyCard extends CardDef {}', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/CardDef/my-card.gts'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + assert.strictEqual( + captured!.headers['content-type'], + 'application/vnd.card+source', + ); + assert.strictEqual( + captured!.body, + 'export class MyCard extends CardDef {}', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-delete sends correct DELETE with Authorization header', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(204, null); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-delete', + toolArgs: { + 'realm-url': realmUrl, + path: 'Card/old-card.json', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'DELETE'); + assert.strictEqual(captured!.url, '/user/target/Card/old-card.json'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-search sends correct QUERY to _search with JSON body', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { data: [] }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let query = JSON.stringify({ + filter: { + type: { module: 'https://example.test/ticket', name: 'Ticket' }, + }, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': realmUrl, + query, + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'QUERY'); + assert.strictEqual(captured!.url, '/user/target/_search'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + assert.strictEqual(captured!.headers.accept, 'application/vnd.card+json'); + assert.strictEqual(captured!.headers['content-type'], 'application/json'); + assert.strictEqual(captured!.body, query); + } finally { + await stopServer(server); + } + }); + + test('realm-atomic sends correct POST to _atomic with JSON:API operations', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { ok: true }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let ops = [ + { op: 'add', href: './CardDef/new.gts', data: { type: 'module' } }, + { op: 'remove', href: './Card/old.json' }, + ]; + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-atomic', + toolArgs: { + 'realm-url': realmUrl, + operations: JSON.stringify(ops), + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_atomic'); + assert.strictEqual( + captured!.headers['content-type'], + 'application/vnd.api+json', + ); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + + let body = JSON.parse(captured!.body); + assert.deepEqual(body['atomic:operations'], ops); + } finally { + await stopServer(server); + } + }); + + test('realm-auth sends correct POST to _realm-auth with Authorization header', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { ok: true }); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-server-jwt-xyz', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-auth', + toolArgs: { + 'realm-server-url': `${origin}/user/target/`, + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_realm-auth'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-server-jwt-xyz', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-create sends correct JSON:API POST to _create-realm with realm-server JWT', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(201, { + data: { type: 'realm', id: `${origin}/user/new-realm/` }, + }); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-server-jwt-minted', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-create', + toolArgs: { + 'realm-server-url': `${origin}/user/target/`, + name: 'New Realm', + endpoint: 'user/new-realm', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_create-realm'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-server-jwt-minted', + ); + assert.strictEqual(captured!.headers.accept, 'application/vnd.api+json'); + assert.strictEqual( + captured!.headers['content-type'], + 'application/vnd.api+json', + ); + + let body = JSON.parse(captured!.body); + assert.strictEqual(body.data.type, 'realm'); + assert.strictEqual(body.data.attributes.name, 'New Realm'); + assert.strictEqual(body.data.attributes.endpoint, 'user/new-realm'); + assert.ok(body.data.attributes.iconURL, 'body includes iconURL'); + assert.ok( + body.data.attributes.backgroundURL, + 'body includes backgroundURL', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-server-session sends OpenID token and returns JWT from Authorization header', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(201, null, { + Authorization: 'Bearer freshly-minted-jwt', + }); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-server-session', + toolArgs: { + 'realm-server-url': `${origin}/user/target/`, + 'openid-token': 'matrix-openid-access-token', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_server-session'); + assert.strictEqual(captured!.headers['content-type'], 'application/json'); + + let body = JSON.parse(captured!.body); + assert.strictEqual( + body.access_token, + 'matrix-openid-access-token', + 'sends OpenID token in request body', + ); + + assert.deepEqual( + result.output, + { token: 'Bearer freshly-minted-jwt' }, + 'captures JWT from Authorization response header', + ); + } finally { + await stopServer(server); + } + }); + + test('end-to-end: realm-server-session → realm-create flow', async function (assert) { + let requests: CapturedRequest[] = []; + + let { server, origin } = await startTestServer((req, respond) => { + requests.push(req); + + if (req.url?.endsWith('_server-session')) { + respond(201, null, { + Authorization: 'Bearer e2e-realm-server-jwt', + }); + } else if (req.url?.endsWith('_create-realm')) { + respond(201, { + data: { type: 'realm', id: `${origin}/user/e2e-scratch/` }, + }); + } else { + respond(404, { error: 'not found' }); + } + }); + + try { + let registry = new ToolRegistry(); + let serverUrl = `${origin}/user/target/`; + + // Step 1: Obtain realm-server JWT + let sessionExecutor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: serverUrl, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + let sessionResult = await sessionExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-server-session', + toolArgs: { + 'realm-server-url': serverUrl, + 'openid-token': 'e2e-openid-token', + }, + }); + + assert.strictEqual(sessionResult.exitCode, 0); + let jwt = (sessionResult.output as { token: string }).token; + assert.strictEqual(jwt, 'Bearer e2e-realm-server-jwt'); + + // Step 2: Use JWT to create a realm + let createExecutor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: serverUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: jwt, + }); + + let createResult = await createExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-create', + toolArgs: { + 'realm-server-url': serverUrl, + name: 'E2E Scratch', + endpoint: 'user/e2e-scratch', + }, + }); + + assert.strictEqual(createResult.exitCode, 0); + assert.strictEqual(requests.length, 2, 'two requests made'); + + // Verify the create request used the minted JWT + let createReq = requests[1]; + assert.strictEqual(createReq.method, 'POST'); + assert.strictEqual(createReq.url, '/user/target/_create-realm'); + assert.strictEqual( + createReq.headers.authorization, + 'Bearer e2e-realm-server-jwt', + 'create request uses the JWT from session', + ); + } finally { + await stopServer(server); + } + }); +}); + +// --------------------------------------------------------------------------- +// Integration tests: safety constraints prevent requests from reaching server +// --------------------------------------------------------------------------- + +module('factory-tool-executor integration > safety constraints', function () { + test('unregistered tool is rejected before any HTTP request', async function (assert) { + let requestCount = 0; + + let { server, origin } = await startTestServer((_req, respond) => { + requestCount++; + respond(200, {}); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'shell-exec-arbitrary', + toolArgs: { command: 'rm -rf /' }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + (err as Error).message.includes('Unregistered tool'), + 'throws for unregistered tool', + ); + } + + assert.strictEqual(requestCount, 0, 'server received zero requests'); + } finally { + await stopServer(server); + } + }); + + test('source realm targeting is rejected before any HTTP request', async function (assert) { + let requestCount = 0; + + let { server, origin } = await startTestServer((_req, respond) => { + requestCount++; + respond(200, {}); + }); + + try { + let registry = new ToolRegistry(); + let sourceUrl = `${origin}/user/source/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + sourceRealmUrl: sourceUrl, + }); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'search-realm', + toolArgs: { realm: sourceUrl }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + (err as Error).message.includes('source realm'), + 'throws for source realm targeting', + ); + } + + assert.strictEqual(requestCount, 0, 'server received zero requests'); + } finally { + await stopServer(server); + } + }); + + test('unknown realm targeting is rejected before any HTTP request', async function (assert) { + let requestCount = 0; + + let { server, origin } = await startTestServer((_req, respond) => { + requestCount++; + respond(200, {}); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': 'https://evil.example.test/hacker/realm/', + path: 'secrets.json', + }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + (err as Error).message.includes('not in the allowed list'), + 'throws for unknown realm', + ); + } + + assert.strictEqual(requestCount, 0, 'server received zero requests'); + } finally { + await stopServer(server); + } + }); +}); diff --git a/packages/software-factory/tests/factory-tool-executor.spec.ts b/packages/software-factory/tests/factory-tool-executor.spec.ts new file mode 100644 index 00000000000..e58fdf11512 --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.spec.ts @@ -0,0 +1,95 @@ +/** + * Live integration tests for the ToolExecutor against the software-factory + * harness realm server. + * + * These run as Playwright specs so they share the harness lifecycle + * (global-setup starts serve:support + cache:prepare, fixtures start + * serve:realm per spec). No browser is needed — these are pure Node tests + * that happen to use the Playwright test runner for harness management. + */ + +import { test } from './fixtures'; +import { expect } from '@playwright/test'; + +import { + ToolExecutor, + ToolNotFoundError, +} from '../scripts/lib/factory-tool-executor'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +test('realm-read fetches .realm.json from the test realm', async ({ + realm, +}) => { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: realm.realmURL.href, + testRealmUrl: realm.realmURL.href, + allowedRealmPrefixes: [realm.realmURL.origin + '/'], + authorization: `Bearer ${realm.ownerBearerToken}`, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': realm.realmURL.href, + path: '.realm.json', + }, + }); + + expect(result.exitCode).toBe(0); + expect(typeof result.output).toBe('object'); +}); + +test('realm-search returns results from the test realm', async ({ realm }) => { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: realm.realmURL.href, + testRealmUrl: realm.realmURL.href, + allowedRealmPrefixes: [realm.realmURL.origin + '/'], + authorization: `Bearer ${realm.ownerBearerToken}`, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': realm.realmURL.href, + query: JSON.stringify({ + filter: { + type: { + module: 'https://cardstack.com/base/card-api', + name: 'CardDef', + }, + }, + page: { size: 1 }, + }), + }, + }); + + expect(result.exitCode).toBe(0); + let output = result.output as { data?: unknown[] }; + expect(Array.isArray(output.data)).toBe(true); +}); + +test('unregistered tool is rejected without reaching the server', async ({ + realm, +}) => { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: realm.realmURL.href, + testRealmUrl: realm.realmURL.href, + authorization: `Bearer ${realm.ownerBearerToken}`, + }); + + await expect( + executor.execute({ + type: 'invoke_tool', + tool: 'shell-exec-arbitrary', + toolArgs: { command: 'rm -rf /' }, + }), + ).rejects.toThrow(ToolNotFoundError); +}); diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts new file mode 100644 index 00000000000..8c2dc8da90b --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -0,0 +1,1267 @@ +import { module, test } from 'qunit'; + +import type { AgentAction, ToolResult } from '../scripts/lib/factory-agent'; +import { + ToolExecutor, + ToolNotFoundError, + ToolSafetyError, + ToolTimeoutError, + type ToolExecutionLogEntry, + type ToolExecutorConfig, +} from '../scripts/lib/factory-tool-executor'; +import { iconURLFor } from '@cardstack/runtime-common/realm-display-defaults'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeConfig( + overrides?: Partial, +): ToolExecutorConfig { + return { + packageRoot: '/fake/software-factory', + targetRealmUrl: 'https://realms.example.test/user/target/', + testRealmUrl: 'https://realms.example.test/user/target-tests/', + ...overrides, + }; +} + +function makeInvokeToolAction( + tool: string, + toolArgs?: Record, +): AgentAction { + return { + type: 'invoke_tool', + tool, + toolArgs, + }; +} + +// --------------------------------------------------------------------------- +// Unregistered tool rejection +// --------------------------------------------------------------------------- + +module('factory-tool-executor > unregistered tool rejection', function () { + test('rejects invoke_tool with empty tool name', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute({ type: 'invoke_tool' } as AgentAction); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolNotFoundError); + } + }); + + test('rejects unregistered tool', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute(makeInvokeToolAction('rm-rf-everything')); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolNotFoundError); + assert.true((err as Error).message.includes('rm-rf-everything')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Argument validation +// --------------------------------------------------------------------------- + +module('factory-tool-executor > argument validation', function () { + test('rejects missing required arguments', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute(makeInvokeToolAction('search-realm', {})); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof Error); + assert.true((err as Error).message.includes('realm')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Safety: source realm protection +// --------------------------------------------------------------------------- + +module('factory-tool-executor > source realm protection', function () { + test('rejects tool targeting source realm', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('search-realm', { + realm: 'https://realms.example.test/user/source/', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('source realm')); + } + }); + + test('rejects source realm without trailing slash', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('search-realm', { + realm: 'https://realms.example.test/user/source', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + } + }); + + test('allows tool targeting target realm', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + fetch: createMockFetch(200, { data: [] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/my-card.gts', + }), + ); + + assert.strictEqual(result.exitCode, 0); + }); + + test('allows tool targeting test realm', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + fetch: createMockFetch(200, { data: [] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target-tests/', + path: 'Test/spec.ts', + }), + ); + + assert.strictEqual(result.exitCode, 0); + }); + + test('rejects realm-api tool targeting unknown realm', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/other/unrelated/', + path: 'foo.json', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('not in the allowed list')); + } + }); + + test('allows realm-api tool targeting scratch realm via prefix', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + allowedRealmPrefixes: ['https://realms.example.test/user/scratch-'], + fetch: createMockFetch(200, { ok: true }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/scratch-123/', + path: 'foo.json', + }), + ); + + assert.strictEqual(result.exitCode, 0); + }); + + test('rejects unknown realm even without sourceRealmUrl configured', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + // No sourceRealmUrl — safety should still enforce allowed-realm targeting + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://evil.example.test/hacker/realm/', + path: 'secrets.json', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('not in the allowed list')); + } + }); + + test('rejects realm-server-url targeting unknown origin', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute( + makeInvokeToolAction('realm-server-session', { + 'realm-server-url': 'https://evil.example.test/', + 'openid-token': 'token', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true( + (err as Error).message.includes('not in the allowed origins'), + ); + } + }); + + test('rejects script tool targeting unknown realm URL', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute( + makeInvokeToolAction('search-realm', { + realm: 'https://evil.example.test/hacker/realm/', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('not in the allowed list')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Realm API execution +// --------------------------------------------------------------------------- + +module('factory-tool-executor > realm-api execution', function () { + test('realm-read makes GET request', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ card: 'data' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/my-card.gts', + }), + ); + + assert.strictEqual(capturedMethod, 'GET'); + assert.strictEqual( + capturedUrl, + 'https://realms.example.test/user/target/CardDef/my-card.gts', + ); + assert.strictEqual(result.exitCode, 0); + assert.deepEqual(result.output, { card: 'data' }); + assert.strictEqual(result.tool, 'realm-read'); + assert.strictEqual(typeof result.durationMs, 'number'); + }); + + test('realm-write makes POST request', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-write', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/new-card.gts', + content: 'export class NewCard {}', + }), + ); + + assert.strictEqual(capturedMethod, 'POST'); + assert.strictEqual( + capturedUrl, + 'https://realms.example.test/user/target/CardDef/new-card.gts', + ); + assert.strictEqual(capturedBody, 'export class NewCard {}'); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-delete makes DELETE request', async function (assert) { + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedMethod = init?.method; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-delete', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/old-card.gts', + }), + ); + + assert.strictEqual(capturedMethod, 'DELETE'); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-search makes QUERY request', async function (assert) { + let capturedMethod: string | undefined; + let capturedUrl: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ data: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-search', { + 'realm-url': 'https://realms.example.test/user/target/', + query: JSON.stringify({ filter: { type: { name: 'Ticket' } } }), + }), + ); + + assert.strictEqual(capturedMethod, 'QUERY'); + assert.true(capturedUrl!.endsWith('_search')); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-atomic makes POST to _atomic endpoint', async function (assert) { + let capturedUrl: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let ops = [{ op: 'add', href: './Foo/bar.json', data: {} }]; + let result = await executor.execute( + makeInvokeToolAction('realm-atomic', { + 'realm-url': 'https://realms.example.test/user/target/', + operations: JSON.stringify(ops), + }), + ); + + assert.true(capturedUrl!.endsWith('_atomic')); + let body = JSON.parse(capturedBody!); + assert.deepEqual(body['atomic:operations'], ops); + assert.strictEqual(result.exitCode, 0); + }); + + test('non-ok response produces exitCode 1', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(404, { error: 'Not found' }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'missing.json', + }), + ); + + assert.strictEqual(result.exitCode, 1); + assert.deepEqual( + (result.output as Record).error, + 'HTTP 404', + ); + }); + + test('includes authorization header when configured', async function (assert) { + let capturedHeaders: Headers | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + authorization: 'Bearer test-token-123', + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = new Headers(init?.headers as HeadersInit); + return new Response(JSON.stringify({}), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + assert.strictEqual( + capturedHeaders!.get('Authorization'), + 'Bearer test-token-123', + ); + }); + + test('realm-auth makes POST to _realm-auth', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-auth', { + 'realm-server-url': 'https://realms.example.test/user/target/', + }), + ); + + assert.strictEqual(capturedMethod, 'POST'); + assert.true(capturedUrl!.endsWith('_realm-auth')); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-create makes POST to _create-realm', async function (assert) { + let capturedUrl: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch-123/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'my-scratch-realm', + endpoint: 'user/scratch-123', + }), + ); + + assert.strictEqual( + capturedUrl, + 'https://realms.example.test/_create-realm', + ); + let body = JSON.parse(capturedBody!); + assert.strictEqual(body.data.type, 'realm'); + assert.strictEqual(body.data.attributes.name, 'my-scratch-realm'); + assert.strictEqual(body.data.attributes.endpoint, 'user/scratch-123'); + assert.ok(body.data.attributes.iconURL, 'iconURL is present'); + assert.ok(body.data.attributes.backgroundURL, 'backgroundURL is present'); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-create with explicit iconURL and backgroundURL', async function (assert) { + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'my-realm', + endpoint: 'user/scratch', + iconURL: 'https://example.test/icon.png', + backgroundURL: 'https://example.test/bg.jpg', + }), + ); + + let body = JSON.parse(capturedBody!); + assert.strictEqual( + body.data.attributes.iconURL, + 'https://example.test/icon.png', + ); + assert.strictEqual( + body.data.attributes.backgroundURL, + 'https://example.test/bg.jpg', + ); + }); + + test('realm-create applies default icon from name', async function (assert) { + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'My Realm', + endpoint: 'user/scratch', + }), + ); + + let body = JSON.parse(capturedBody!); + assert.strictEqual( + body.data.attributes.iconURL, + iconURLFor('My Realm'), + 'iconURL defaults from name', + ); + }); + + test('realm-create applies default random background', async function (assert) { + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'My Realm', + endpoint: 'user/scratch', + }), + ); + + let body = JSON.parse(capturedBody!); + assert.true( + body.data.attributes.backgroundURL.startsWith( + 'https://boxel-images.boxel.ai/background-images/', + ), + 'backgroundURL defaults to a random background', + ); + }); + + test('realm-create updates Matrix account data when config present', async function (assert) { + let fetchCalls: { url: string; method: string }[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + matrixUrl: 'https://matrix.example.test', + matrixAccessToken: 'matrix-token-123', + matrixUserId: '@factory:example.test', + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + let url = String(input); + let method = init?.method ?? 'GET'; + fetchCalls.push({ url, method }); + + if (url.includes('_create-realm')) { + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + } + if (url.includes('account_data') && method === 'GET') { + return new Response( + JSON.stringify({ realms: ['https://existing.test/'] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + } + // PUT account_data + return new Response('{}', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'scratch', + endpoint: 'user/scratch', + }), + ); + + assert.strictEqual(fetchCalls.length, 3, 'three fetch calls made'); + assert.true( + fetchCalls[0].url.includes('_create-realm'), + 'first call is _create-realm', + ); + assert.true( + fetchCalls[1].url.includes('account_data'), + 'second call is Matrix GET account_data', + ); + assert.strictEqual(fetchCalls[1].method, 'GET'); + assert.true( + fetchCalls[2].url.includes('account_data'), + 'third call is Matrix PUT account_data', + ); + assert.strictEqual(fetchCalls[2].method, 'PUT'); + }); + + test('realm-create skips Matrix update when config absent', async function (assert) { + let fetchCalls: { url: string; method: string }[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + // No matrixUrl, matrixAccessToken, or matrixUserId + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + let url = String(input); + let method = init?.method ?? 'GET'; + fetchCalls.push({ url, method }); + + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'scratch', + endpoint: 'user/scratch', + }), + ); + + assert.strictEqual( + fetchCalls.length, + 1, + 'only one fetch call (no Matrix update)', + ); + assert.true( + fetchCalls[0].url.includes('_create-realm'), + 'only call is _create-realm', + ); + }); + + test('realm-server-session sends OpenID token and captures Authorization header', async function (assert) { + let capturedUrl: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(null, { + status: 201, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer realm-server-jwt-123', + }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-server-session', { + 'realm-server-url': 'https://realms.example.test/user/target/', + 'openid-token': 'openid-access-token-xyz', + }), + ); + + assert.true(capturedUrl!.endsWith('_server-session')); + let body = JSON.parse(capturedBody!); + assert.strictEqual( + body.access_token, + 'openid-access-token-xyz', + 'sends OpenID token in request body', + ); + assert.strictEqual(result.exitCode, 0); + assert.deepEqual( + result.output, + { token: 'Bearer realm-server-jwt-123' }, + 'captures Authorization header in output', + ); + }); +}); + +// --------------------------------------------------------------------------- +// Auth header propagation +// --------------------------------------------------------------------------- + +module('factory-tool-executor > auth header propagation', function () { + function createHeaderCapturingFetch(): { + fetch: typeof globalThis.fetch; + getCapturedHeaders: () => Headers | undefined; + } { + let capturedHeaders: Headers | undefined; + return { + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = new Headers(init?.headers as HeadersInit); + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + getCapturedHeaders: () => capturedHeaders, + }; + } + + test('realm-read sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/foo.json', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-write sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-write', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/new.gts', + content: 'export class NewCard {}', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-delete sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-delete', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/old.json', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-search sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-search', { + 'realm-url': 'https://realms.example.test/user/target/', + query: '{}', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-atomic sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-atomic', { + 'realm-url': 'https://realms.example.test/user/target/', + operations: '[]', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-create sends realm-server JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + authorization: 'Bearer realm-server-jwt-xyz', + fetch, + }), + ); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/user/target/', + name: 'scratch', + endpoint: 'user/scratch', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-server-jwt-xyz', + ); + }); + + test('realm-auth sends server JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-auth', { + 'realm-server-url': 'https://realms.example.test/user/target/', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('no Authorization header when authorization is not configured', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ fetch }), // no authorization + ); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/foo.json', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + null, + 'no Authorization header sent', + ); + }); + + test('realm-server-session JWT can be used for subsequent realm-create', async function (assert) { + let registry = new ToolRegistry(); + + // Step 1: Get realm server JWT via realm-server-session + let sessionExecutor = new ToolExecutor( + registry, + makeConfig({ + fetch: (async () => { + return new Response(null, { + status: 201, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer minted-realm-server-jwt', + }, + }); + }) as typeof globalThis.fetch, + }), + ); + + let sessionResult = await sessionExecutor.execute( + makeInvokeToolAction('realm-server-session', { + 'realm-server-url': 'https://realms.example.test/user/target/', + 'openid-token': 'matrix-openid-token', + }), + ); + + let jwt = (sessionResult.output as { token: string }).token; + assert.strictEqual(jwt, 'Bearer minted-realm-server-jwt'); + + // Step 2: Use the JWT for realm-create + let capturedHeaders: Headers | undefined; + let createExecutor = new ToolExecutor( + registry, + makeConfig({ + authorization: jwt, + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = new Headers(init?.headers as HeadersInit); + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }), + ); + + await createExecutor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/user/target/', + name: 'test-realm', + endpoint: 'user/test-realm', + }), + ); + + assert.strictEqual( + capturedHeaders!.get('Authorization'), + 'Bearer minted-realm-server-jwt', + 'realm-create uses the JWT from realm-server-session', + ); + }); +}); + +// --------------------------------------------------------------------------- +// Logging +// --------------------------------------------------------------------------- + +module('factory-tool-executor > logging', function () { + test('logs successful tool execution', async function (assert) { + let logEntries: ToolExecutionLogEntry[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(200, { data: [] }), + log: (entry) => logEntries.push(entry), + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + assert.strictEqual(logEntries.length, 1); + assert.strictEqual(logEntries[0].tool, 'realm-read'); + assert.strictEqual(logEntries[0].category, 'realm-api'); + assert.strictEqual(logEntries[0].exitCode, 0); + assert.strictEqual(typeof logEntries[0].durationMs, 'number'); + assert.strictEqual(logEntries[0].error, undefined); + }); + + test('logs failed tool execution', async function (assert) { + let logEntries: ToolExecutionLogEntry[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(500, { error: 'Internal error' }), + log: (entry) => logEntries.push(entry), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'broken.json', + }), + ); + + assert.strictEqual(result.exitCode, 1); + assert.strictEqual(logEntries.length, 1); + assert.strictEqual(logEntries[0].exitCode, 1); + }); +}); + +// --------------------------------------------------------------------------- +// ToolResult serialization +// --------------------------------------------------------------------------- + +module('factory-tool-executor > ToolResult shape', function () { + test('successful result has expected shape', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(200, { cards: ['a', 'b'] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + assert.strictEqual(typeof result.tool, 'string'); + assert.strictEqual(typeof result.exitCode, 'number'); + assert.strictEqual(typeof result.durationMs, 'number'); + assert.notStrictEqual(result.output, undefined, 'output is defined'); + }); + + test('ToolResult can be serialized to JSON', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(200, { data: [1, 2, 3] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + let serialized = JSON.stringify(result); + let deserialized = JSON.parse(serialized) as ToolResult; + assert.strictEqual(deserialized.tool, 'realm-read'); + assert.strictEqual(deserialized.exitCode, 0); + assert.deepEqual(deserialized.output, { data: [1, 2, 3] }); + }); +}); + +// --------------------------------------------------------------------------- +// Timeout behavior +// --------------------------------------------------------------------------- + +module('factory-tool-executor > timeout', function () { + test('realm-api call times out with ToolTimeoutError', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + timeoutMs: 50, + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + // Respect the AbortSignal so the timeout mechanism works + return new Promise((resolve, reject) => { + let signal = init?.signal; + if (signal) { + signal.addEventListener('abort', () => { + reject( + new DOMException('The operation was aborted.', 'AbortError'), + ); + }); + } + // Never resolves on its own within timeout + setTimeout( + () => + resolve( + new Response('{}', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ), + 5000, + ); + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + try { + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'slow.json', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolTimeoutError); + assert.true((err as Error).message.includes('50ms')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function createMockFetch( + status: number, + body: unknown, +): typeof globalThis.fetch { + return (async () => { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} diff --git a/packages/software-factory/tests/factory-tool-registry.test.ts b/packages/software-factory/tests/factory-tool-registry.test.ts new file mode 100644 index 00000000000..d3fc3f6350f --- /dev/null +++ b/packages/software-factory/tests/factory-tool-registry.test.ts @@ -0,0 +1,310 @@ +import { module, test } from 'qunit'; + +import { + BOXEL_CLI_TOOLS, + getDefaultToolRegistry, + REALM_API_TOOLS, + SCRIPT_TOOLS, + ToolRegistry, + type ToolManifest, +} from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// ToolRegistry construction +// --------------------------------------------------------------------------- + +module('factory-tool-registry > ToolRegistry construction', function () { + test('default registry includes all built-in tools', function (assert) { + let registry = new ToolRegistry(); + let expectedCount = + SCRIPT_TOOLS.length + BOXEL_CLI_TOOLS.length + REALM_API_TOOLS.length; + assert.strictEqual( + registry.size, + expectedCount, + `registry has ${expectedCount} tools`, + ); + }); + + test('accepts custom manifest list', function (assert) { + let custom: ToolManifest[] = [ + { + name: 'custom-tool', + description: 'A test tool', + category: 'script', + args: [], + outputFormat: 'json', + }, + ]; + let registry = new ToolRegistry(custom); + assert.strictEqual(registry.size, 1); + assert.true(registry.has('custom-tool')); + }); + + test('empty manifest list creates empty registry', function (assert) { + let registry = new ToolRegistry([]); + assert.strictEqual(registry.size, 0); + }); + + test('throws on duplicate tool names', function (assert) { + let dupes: ToolManifest[] = [ + { + name: 'same-name', + description: 'first', + category: 'script', + args: [], + outputFormat: 'json', + }, + { + name: 'same-name', + description: 'second', + category: 'script', + args: [], + outputFormat: 'json', + }, + ]; + assert.throws( + () => new ToolRegistry(dupes), + (err: Error) => + err.message.includes('Duplicate') && err.message.includes('same-name'), + ); + }); +}); + +// --------------------------------------------------------------------------- +// getManifests +// --------------------------------------------------------------------------- + +module('factory-tool-registry > getManifests', function () { + test('returns all manifests', function (assert) { + let registry = new ToolRegistry(); + let manifests = registry.getManifests(); + assert.true(manifests.length > 0, 'returns non-empty array'); + assert.true(Array.isArray(manifests), 'returns an array'); + }); + + test('returned array is a copy (not internal state)', function (assert) { + let registry = new ToolRegistry(); + let a = registry.getManifests(); + let b = registry.getManifests(); + assert.notStrictEqual(a, b, 'different array instances'); + assert.deepEqual(a, b, 'same contents'); + }); +}); + +// --------------------------------------------------------------------------- +// getManifest +// --------------------------------------------------------------------------- + +module('factory-tool-registry > getManifest', function () { + test('returns manifest for known tool', function (assert) { + let registry = new ToolRegistry(); + let manifest = registry.getManifest('search-realm'); + assert.ok(manifest, 'manifest is defined'); + assert.strictEqual(manifest!.name, 'search-realm'); + assert.strictEqual(manifest!.category, 'script'); + }); + + test('returns undefined for unknown tool', function (assert) { + let registry = new ToolRegistry(); + assert.strictEqual(registry.getManifest('nonexistent'), undefined); + }); +}); + +// --------------------------------------------------------------------------- +// has +// --------------------------------------------------------------------------- + +module('factory-tool-registry > has', function () { + test('returns true for registered tool', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('search-realm')); + assert.true(registry.has('boxel-sync')); + assert.true(registry.has('realm-read')); + }); + + test('returns false for unregistered tool', function (assert) { + let registry = new ToolRegistry(); + assert.false(registry.has('rm-rf')); + assert.false(registry.has('')); + }); +}); + +// --------------------------------------------------------------------------- +// validateArgs +// --------------------------------------------------------------------------- + +module('factory-tool-registry > validateArgs', function () { + test('returns empty array for valid args', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { + realm: 'http://example.test/', + }); + assert.deepEqual(errors, []); + }); + + test('returns error for missing required arg', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', {}); + assert.strictEqual(errors.length, 1); + assert.true(errors[0].includes('realm')); + }); + + test('returns error for unknown tool', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('nonexistent', {}); + assert.strictEqual(errors.length, 1); + assert.true(errors[0].includes('Unknown tool')); + }); + + test('multiple missing required args produce multiple errors', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('realm-write', {}); + assert.true( + errors.length >= 3, + `expected at least 3 errors, got ${errors.length}`, + ); + }); + + test('optional args do not produce errors when missing', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { + realm: 'http://example.test/', + }); + assert.deepEqual(errors, [], 'no errors for missing optional args'); + }); + + test('empty string for required arg produces error', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { realm: '' }); + assert.strictEqual(errors.length, 1); + }); + + test('whitespace-only string for required arg produces error', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { realm: ' ' }); + assert.strictEqual(errors.length, 1, 'whitespace-only value is rejected'); + }); +}); + +// --------------------------------------------------------------------------- +// Built-in manifest completeness +// --------------------------------------------------------------------------- + +module('factory-tool-registry > built-in manifests', function () { + test('all script tools have correct category', function (assert) { + for (let tool of SCRIPT_TOOLS) { + assert.strictEqual( + tool.category, + 'script', + `${tool.name} has category "script"`, + ); + } + }); + + test('all boxel-cli tools have correct category', function (assert) { + for (let tool of BOXEL_CLI_TOOLS) { + assert.strictEqual( + tool.category, + 'boxel-cli', + `${tool.name} has category "boxel-cli"`, + ); + } + }); + + test('all realm-api tools have correct category', function (assert) { + for (let tool of REALM_API_TOOLS) { + assert.strictEqual( + tool.category, + 'realm-api', + `${tool.name} has category "realm-api"`, + ); + } + }); + + test('all tools have unique names', function (assert) { + let registry = new ToolRegistry(); + let manifests = registry.getManifests(); + let names = manifests.map((m) => m.name); + let unique = new Set(names); + assert.strictEqual(unique.size, names.length, 'all tool names are unique'); + }); + + test('all tools have non-empty description', function (assert) { + let registry = new ToolRegistry(); + for (let manifest of registry.getManifests()) { + assert.true( + manifest.description.length > 0, + `${manifest.name} has a description`, + ); + } + }); + + test('all tool args have required fields', function (assert) { + let registry = new ToolRegistry(); + for (let manifest of registry.getManifests()) { + for (let arg of manifest.args) { + assert.true(arg.name.length > 0, `${manifest.name} arg has a name`); + assert.true( + arg.type.length > 0, + `${manifest.name}:${arg.name} has a type`, + ); + assert.true( + arg.description.length > 0, + `${manifest.name}:${arg.name} has a description`, + ); + assert.strictEqual( + typeof arg.required, + 'boolean', + `${manifest.name}:${arg.name} has required flag`, + ); + } + } + }); + + test('expected script tools are registered', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('search-realm')); + assert.true(registry.has('pick-ticket')); + assert.true(registry.has('get-session')); + assert.true(registry.has('run-realm-tests')); + }); + + test('expected boxel-cli tools are registered', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('boxel-sync')); + assert.true(registry.has('boxel-push')); + assert.true(registry.has('boxel-pull')); + assert.true(registry.has('boxel-status')); + assert.true(registry.has('boxel-create')); + assert.true(registry.has('boxel-history')); + }); + + test('expected realm-api tools are registered', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('realm-read')); + assert.true(registry.has('realm-write')); + assert.true(registry.has('realm-delete')); + assert.true(registry.has('realm-atomic')); + assert.true(registry.has('realm-search')); + assert.true(registry.has('realm-create')); + assert.true(registry.has('realm-server-session')); + assert.true(registry.has('realm-auth')); + }); +}); + +// --------------------------------------------------------------------------- +// getDefaultToolRegistry +// --------------------------------------------------------------------------- + +module('factory-tool-registry > getDefaultToolRegistry', function () { + test('returns a ToolRegistry instance', function (assert) { + let registry = getDefaultToolRegistry(); + assert.true(registry instanceof ToolRegistry); + }); + + test('returns the same instance on repeated calls', function (assert) { + let a = getDefaultToolRegistry(); + let b = getDefaultToolRegistry(); + assert.strictEqual(a, b, 'same singleton instance'); + }); +}); diff --git a/packages/software-factory/tests/index.ts b/packages/software-factory/tests/index.ts index dcf22a7c950..cc1376f0986 100644 --- a/packages/software-factory/tests/index.ts +++ b/packages/software-factory/tests/index.ts @@ -6,4 +6,7 @@ import './factory-entrypoint.test'; import './factory-entrypoint.integration.test'; import './factory-skill-loader.test'; import './factory-target-realm.test'; +import './factory-tool-executor.test'; +import './factory-tool-executor.integration.test'; +import './factory-tool-registry.test'; import './realm-auth.test';