From 03e4058c4071478352a194d2ae70dcfefddf3846 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 16:07:25 -0400 Subject: [PATCH 01/13] Implement tool registry and executor for agent actions (CS-10479) Add ToolRegistry with 19 built-in tool manifests across three categories (script, boxel-cli, realm-api) and ToolExecutor with dispatch logic, safety constraints, timeout handling, and audit logging. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/software-factory/package.json | 1 + .../scripts/factory-tools-smoke.ts | 255 ++++++ .../scripts/lib/factory-tool-executor.ts | 762 ++++++++++++++++++ .../scripts/lib/factory-tool-registry.ts | 595 ++++++++++++++ .../tests/factory-tool-executor.test.ts | 681 ++++++++++++++++ .../tests/factory-tool-registry.test.ts | 281 +++++++ packages/software-factory/tests/index.ts | 2 + 7 files changed, 2577 insertions(+) create mode 100644 packages/software-factory/scripts/factory-tools-smoke.ts create mode 100644 packages/software-factory/scripts/lib/factory-tool-executor.ts create mode 100644 packages/software-factory/scripts/lib/factory-tool-registry.ts create mode 100644 packages/software-factory/tests/factory-tool-executor.test.ts create mode 100644 packages/software-factory/tests/factory-tool-registry.test.ts diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index 0aac11f2fcf..c64335dd371 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -11,6 +11,7 @@ "cache:prepare": "NODE_NO_WARNINGS=1 ts-node --transpileOnly src/cli/cache-realm.ts", "factory:agent-smoke": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/factory-agent-smoke.ts", "factory:go": "NODE_NO_WARNINGS=1 ts-node --transpileOnly src/cli/factory-entrypoint.ts", + "factory:tools-smoke": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/factory-tools-smoke.ts", "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"", "lint:js": "eslint . --report-unused-disable-directives --cache", diff --git a/packages/software-factory/scripts/factory-tools-smoke.ts b/packages/software-factory/scripts/factory-tools-smoke.ts new file mode 100644 index 00000000000..ac89a0daae5 --- /dev/null +++ b/packages/software-factory/scripts/factory-tools-smoke.ts @@ -0,0 +1,255 @@ +/** + * Smoke test for the ToolRegistry and ToolExecutor. + * + * No running services required — exercises the registry, manifest validation, + * safety checks, and a mocked realm-api round-trip entirely in-process. + * + * Usage: + * pnpm factory:tools-smoke + */ + +import type { AgentAction } from './lib/factory-agent'; +import { + ToolExecutor, + ToolNotFoundError, + ToolSafetyError, +} from './lib/factory-tool-executor'; +import { ToolRegistry } from './lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +let passed = 0; +let failed = 0; + +function check(label: string, ok: boolean, detail?: string): void { + if (ok) { + passed++; + console.log(` \u2713 ${label}`); + } else { + failed++; + console.log(` \u2717 ${label}${detail ? ` -- ${detail}` : ''}`); + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + // ----------------------------------------------------------------------- + // 1. Registry + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Tool Registry ==='); + console.log(''); + + let registry = new ToolRegistry(); + let manifests = registry.getManifests(); + + console.log(`Registered tools: ${manifests.length}`); + console.log(''); + + let byCategory: Record = {}; + for (let m of manifests) { + (byCategory[m.category] ??= []).push(m.name); + } + + for (let [category, names] of Object.entries(byCategory)) { + console.log(` ${category} (${names.length}):`); + for (let name of names) { + let manifest = registry.getManifest(name)!; + let requiredArgs = manifest.args + .filter((a) => a.required) + .map((a) => a.name); + let optionalArgs = manifest.args + .filter((a) => !a.required) + .map((a) => a.name); + console.log( + ` - ${name} [${manifest.outputFormat}]` + + (requiredArgs.length + ? ` required: ${requiredArgs.join(', ')}` + : '') + + (optionalArgs.length ? ` optional: ${optionalArgs.join(', ')}` : ''), + ); + } + console.log(''); + } + + check('has script tools', byCategory['script']?.length === 4); + check('has boxel-cli tools', byCategory['boxel-cli']?.length === 6); + check('has realm-api tools', byCategory['realm-api']?.length === 9); + check( + 'all names unique', + new Set(manifests.map((m) => m.name)).size === manifests.length, + ); + + // ----------------------------------------------------------------------- + // 2. Argument validation + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Argument Validation ==='); + console.log(''); + + let validErrors = registry.validateArgs('search-realm', { + realm: 'http://example.test/', + }); + check('valid args -> no errors', validErrors.length === 0); + + let missingErrors = registry.validateArgs('search-realm', {}); + check( + 'missing required arg -> error', + missingErrors.length > 0 && missingErrors[0].includes('realm'), + ); + + let unknownErrors = registry.validateArgs('not-a-tool', {}); + check( + 'unknown tool -> error', + unknownErrors.length > 0 && unknownErrors[0].includes('Unknown'), + ); + + // ----------------------------------------------------------------------- + // 3. Safety constraints + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Safety Constraints ==='); + console.log(''); + + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: 'https://realms.example.test/user/target/', + testRealmUrl: 'https://realms.example.test/user/target-tests/', + sourceRealmUrl: 'https://realms.example.test/user/source/', + allowedRealmPrefixes: ['https://realms.example.test/user/scratch-'], + }); + + // Unregistered tool + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'rm-rf', + } as AgentAction); + check('rejects unregistered tool', false, 'did not throw'); + } catch (err) { + check('rejects unregistered tool', err instanceof ToolNotFoundError); + } + + // Source realm targeting + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'search-realm', + toolArgs: { realm: 'https://realms.example.test/user/source/' }, + }); + check('rejects source realm', false, 'did not throw'); + } catch (err) { + check('rejects source realm', err instanceof ToolSafetyError); + } + + // Unknown realm targeting (realm-api) + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': 'https://evil.example.test/hacker/realm/', + path: 'secrets.json', + }, + }); + check('rejects unknown realm', false, 'did not throw'); + } catch (err) { + check('rejects unknown realm', err instanceof ToolSafetyError); + } + + // ----------------------------------------------------------------------- + // 4. Mocked realm-api round-trip + // ----------------------------------------------------------------------- + + console.log(''); + console.log('=== Realm API Round-Trip (mock) ==='); + console.log(''); + + let mockCallCount = 0; + + let mockExecutor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: 'https://realms.example.test/user/target/', + testRealmUrl: 'https://realms.example.test/user/target-tests/', + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + mockCallCount++; + let url = String(input); + let method = init?.method ?? 'GET'; + console.log(` -> ${method} ${url}`); + return new Response( + JSON.stringify({ + data: [{ id: 'CardDef/hello', type: 'card' }], + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }) as typeof globalThis.fetch, + }); + + let readResult = await mockExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/hello.gts', + }, + }); + check('realm-read exitCode=0', readResult.exitCode === 0); + check('realm-read has output', readResult.output !== undefined); + check( + `realm-read duration ${readResult.durationMs}ms`, + readResult.durationMs >= 0, + ); + + let searchResult = await mockExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': 'https://realms.example.test/user/target/', + query: JSON.stringify({ filter: { type: { name: 'Ticket' } } }), + }, + }); + check('realm-search exitCode=0', searchResult.exitCode === 0); + + let writeResult = await mockExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-write', + toolArgs: { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/new.gts', + content: 'export class NewCard {}', + }, + }); + check('realm-write exitCode=0', writeResult.exitCode === 0); + + check(`mock fetch called ${mockCallCount} times`, mockCallCount === 3); + + // ----------------------------------------------------------------------- + // Summary + // ----------------------------------------------------------------------- + + console.log(''); + console.log('==========================='); + console.log(` ${passed} passed, ${failed} failed`); + console.log('==========================='); + console.log(''); + + if (failed > 0) { + process.exit(1); + } +} + +main().catch((err: unknown) => { + console.error( + 'Smoke test failed:', + err instanceof Error ? err.message : String(err), + ); + process.exit(1); +}); diff --git a/packages/software-factory/scripts/lib/factory-tool-executor.ts b/packages/software-factory/scripts/lib/factory-tool-executor.ts new file mode 100644 index 00000000000..3a7c7b02300 --- /dev/null +++ b/packages/software-factory/scripts/lib/factory-tool-executor.ts @@ -0,0 +1,762 @@ +import { spawn } from 'node:child_process'; +import { resolve } from 'node:path'; + +import type { AgentAction, ToolResult } from './factory-agent'; +import type { ToolRegistry } from './factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const DEFAULT_TIMEOUT_MS = 60_000; + +/** + * Map from script tool name to the script file that implements it. + * Paths are relative to `packages/software-factory/scripts/`. + */ +const SCRIPT_FILE_MAP: Record = { + 'search-realm': 'boxel-search.ts', + 'pick-ticket': 'pick-ticket.ts', + 'get-session': 'boxel-session.ts', + 'run-realm-tests': 'run-realm-tests.ts', +}; + +/** + * Map from boxel-cli tool name to the `npx boxel` subcommand. + */ +const BOXEL_CLI_COMMAND_MAP: Record = { + 'boxel-sync': 'sync', + 'boxel-push': 'push', + 'boxel-pull': 'pull', + 'boxel-status': 'status', + 'boxel-create': 'create', + 'boxel-history': 'history', +}; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ToolExecutorConfig { + /** Absolute path to the software-factory package root. */ + packageRoot: string; + /** Target realm URL — tools may only target this realm. */ + targetRealmUrl: string; + /** Test realm URL — tools may also target this realm. */ + testRealmUrl: string; + /** Additional scratch realm URL prefixes that are allowed. */ + allowedRealmPrefixes?: string[]; + /** Source realm URL — tools must NEVER target this realm. */ + sourceRealmUrl?: string; + /** Fetch implementation for realm API calls. */ + fetch?: typeof globalThis.fetch; + /** Authorization header value for realm API calls. */ + authorization?: string; + /** Per-invocation timeout in ms (default: 60 000). */ + timeoutMs?: number; + /** Optional log function for auditability. */ + log?: (entry: ToolExecutionLogEntry) => void; +} + +export interface ToolExecutionLogEntry { + tool: string; + category: 'script' | 'boxel-cli' | 'realm-api'; + args: Record; + exitCode: number; + durationMs: number; + error?: string; +} + +// --------------------------------------------------------------------------- +// Error classes +// --------------------------------------------------------------------------- + +export class ToolSafetyError extends Error { + constructor(message: string) { + super(message); + this.name = 'ToolSafetyError'; + } +} + +export class ToolTimeoutError extends Error { + constructor(tool: string, timeoutMs: number) { + super(`Tool "${tool}" timed out after ${timeoutMs}ms`); + this.name = 'ToolTimeoutError'; + } +} + +export class ToolNotFoundError extends Error { + constructor(tool: string) { + super(`Unregistered tool: "${tool}"`); + this.name = 'ToolNotFoundError'; + } +} + +// --------------------------------------------------------------------------- +// ToolExecutor +// --------------------------------------------------------------------------- + +export class ToolExecutor { + private registry: ToolRegistry; + private config: ToolExecutorConfig; + private timeoutMs: number; + + constructor(registry: ToolRegistry, config: ToolExecutorConfig) { + this.registry = registry; + this.config = config; + this.timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; + } + + /** + * Execute a validated `invoke_tool` action and return the result. + * + * The executor: + * 1. Validates the tool name against the registry + * 2. Validates arguments against the manifest + * 3. Enforces safety constraints (no source realm targeting) + * 4. Dispatches to the appropriate sub-executor + * 5. Captures output as a ToolResult + */ + async execute(action: AgentAction): Promise { + let toolName = action.tool; + if (!toolName) { + throw new ToolNotFoundError('(empty)'); + } + + let manifest = this.registry.getManifest(toolName); + if (!manifest) { + throw new ToolNotFoundError(toolName); + } + + let toolArgs = action.toolArgs ?? {}; + + // Validate required args + let argErrors = this.registry.validateArgs(toolName, toolArgs); + if (argErrors.length > 0) { + throw new Error( + `Invalid arguments for tool "${toolName}": ${argErrors.join('; ')}`, + ); + } + + // Safety: reject source realm targeting + this.enforceRealmSafety(toolName, toolArgs); + + let start = Date.now(); + let result: ToolResult; + + try { + switch (manifest.category) { + case 'script': + result = await this.executeScript(toolName, toolArgs); + break; + case 'boxel-cli': + result = await this.executeBoxelCli(toolName, toolArgs); + break; + case 'realm-api': + result = await this.executeRealmApi(toolName, toolArgs); + break; + default: + throw new Error(`Unknown tool category: ${manifest.category}`); + } + } catch (error) { + let durationMs = Date.now() - start; + let errorMessage = error instanceof Error ? error.message : String(error); + + this.logExecution({ + tool: toolName, + category: manifest.category, + args: toolArgs, + exitCode: 1, + durationMs, + error: errorMessage, + }); + + // Re-throw safety and timeout errors as-is + if ( + error instanceof ToolSafetyError || + error instanceof ToolTimeoutError || + error instanceof ToolNotFoundError + ) { + throw error; + } + + return { + tool: toolName, + exitCode: 1, + output: { error: errorMessage }, + durationMs, + }; + } + + this.logExecution({ + tool: toolName, + category: manifest.category, + args: toolArgs, + exitCode: result.exitCode, + durationMs: result.durationMs, + }); + + return result; + } + + // ------------------------------------------------------------------------- + // Safety + // ------------------------------------------------------------------------- + + private enforceRealmSafety( + toolName: string, + toolArgs: Record, + ): void { + let sourceUrl = this.config.sourceRealmUrl; + if (!sourceUrl) { + return; + } + + let normalizedSource = ensureTrailingSlash(sourceUrl); + + // Check all string args that look like realm URLs + let realmArgNames = [ + 'realm', + 'realm-url', + 'realm-server-url', + 'local-dir', + 'path', + ]; + + for (let argName of realmArgNames) { + let value = toolArgs[argName]; + if (typeof value === 'string' && looksLikeUrl(value)) { + let normalizedValue = ensureTrailingSlash(value); + if (normalizedValue === normalizedSource) { + throw new ToolSafetyError( + `Tool "${toolName}" cannot target the source realm: ${sourceUrl}`, + ); + } + } + } + + // For realm-api tools, also check if the target URL is allowed + let manifest = this.registry.getManifest(toolName); + if (manifest?.category === 'realm-api') { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string' && looksLikeUrl(realmUrl)) { + this.validateRealmTarget(toolName, realmUrl); + } + } + + // Extra validation for destructive operations + this.validateDestructiveOps(toolName, toolArgs); + } + + private validateRealmTarget(toolName: string, realmUrl: string): void { + let normalized = ensureTrailingSlash(realmUrl); + let target = ensureTrailingSlash(this.config.targetRealmUrl); + let test = ensureTrailingSlash(this.config.testRealmUrl); + + // Exact realm matches (with trailing slash normalization) + let exactAllowed = [target, test]; + + // Prefix matches (no trailing slash — these are URL path prefixes) + let prefixAllowed = this.config.allowedRealmPrefixes ?? []; + + let isAllowed = + exactAllowed.some((exact) => normalized === exact) || + prefixAllowed.some((prefix) => normalized.startsWith(prefix)); + + if (!isAllowed) { + throw new ToolSafetyError( + `Tool "${toolName}" targets realm "${realmUrl}" which is not in the allowed list. ` + + `Allowed: ${[...exactAllowed, ...prefixAllowed].join(', ')}`, + ); + } + } + + private validateDestructiveOps( + toolName: string, + toolArgs: Record, + ): void { + // realm-delete and realm-atomic with remove ops need extra care + if (toolName === 'realm-delete') { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string') { + this.validateRealmTarget(toolName, realmUrl); + } + } + + if (toolName === 'realm-atomic') { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string') { + this.validateRealmTarget(toolName, realmUrl); + } + } + + // boxel-push with --delete + if (toolName === 'boxel-push' && toolArgs['delete']) { + let realmUrl = toolArgs['realm-url']; + if (typeof realmUrl === 'string' && looksLikeUrl(realmUrl)) { + this.validateRealmTarget(toolName, realmUrl); + } + } + + // realm-create and realm-reindex require extra validation + if (toolName === 'realm-create' || toolName === 'realm-reindex') { + // These are allowed but logged — the orchestrator trusts the agent + // chose them deliberately within the allowed realm set. + } + } + + // ------------------------------------------------------------------------- + // Sub-executors + // ------------------------------------------------------------------------- + + private async executeScript( + toolName: string, + toolArgs: Record, + ): Promise { + let scriptFile = SCRIPT_FILE_MAP[toolName]; + if (!scriptFile) { + throw new Error(`No script file mapped for tool "${toolName}"`); + } + + let scriptPath = resolve(this.config.packageRoot, 'scripts', scriptFile); + + let cliArgs = buildCliArgs(toolArgs); + + return this.spawnProcess( + toolName, + 'npx', + ['ts-node', '--transpileOnly', scriptPath, ...cliArgs], + 'json', + ); + } + + private async executeBoxelCli( + toolName: string, + toolArgs: Record, + ): Promise { + let subcommand = BOXEL_CLI_COMMAND_MAP[toolName]; + if (!subcommand) { + throw new Error(`No boxel-cli command mapped for tool "${toolName}"`); + } + + let cliArgs = buildBoxelCliArgs(toolName, subcommand, toolArgs); + + return this.spawnProcess(toolName, 'npx', ['boxel', ...cliArgs], 'text'); + } + + private async executeRealmApi( + toolName: string, + toolArgs: Record, + ): Promise { + let fetchImpl = this.config.fetch ?? globalThis.fetch; + let start = Date.now(); + + let { url, method, headers, body } = buildRealmApiRequest( + toolName, + toolArgs, + this.config.authorization, + ); + + let controller = new AbortController(); + let timeout = setTimeout(() => controller.abort(), this.timeoutMs); + + try { + let response = await fetchImpl(url, { + method, + headers, + body, + signal: controller.signal, + }); + + let durationMs = Date.now() - start; + let responseBody: unknown; + + let contentType = response.headers.get('content-type') ?? ''; + if (contentType.includes('json')) { + responseBody = await response.json(); + } else { + responseBody = await response.text(); + } + + return { + tool: toolName, + exitCode: response.ok ? 0 : 1, + output: response.ok + ? responseBody + : { + error: `HTTP ${response.status}`, + body: responseBody, + }, + durationMs, + }; + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') { + throw new ToolTimeoutError(toolName, this.timeoutMs); + } + throw error; + } finally { + clearTimeout(timeout); + } + } + + // ------------------------------------------------------------------------- + // Process spawning + // ------------------------------------------------------------------------- + + private spawnProcess( + toolName: string, + command: string, + args: string[], + outputFormat: 'json' | 'text', + ): Promise { + return new Promise((resolvePromise, reject) => { + let start = Date.now(); + let stdout = ''; + let stderr = ''; + + let child = spawn(command, args, { + cwd: this.config.packageRoot, + env: { ...process.env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let timer = setTimeout(() => { + child.kill('SIGTERM'); + // Give the process a moment to clean up, then force kill + setTimeout(() => { + if (!child.killed) { + child.kill('SIGKILL'); + } + }, 5000); + reject(new ToolTimeoutError(toolName, this.timeoutMs)); + }, this.timeoutMs); + + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + + child.on('error', (error: Error) => { + clearTimeout(timer); + reject(error); + }); + + child.on('close', (code: number | null) => { + clearTimeout(timer); + let durationMs = Date.now() - start; + let exitCode = code ?? 1; + + let output: unknown; + if (outputFormat === 'json') { + try { + output = JSON.parse(stdout.trim()); + } catch { + output = { + raw: stdout.trim(), + ...(stderr.trim() ? { stderr: stderr.trim() } : {}), + }; + } + } else { + output = stdout.trim(); + if (stderr.trim() && exitCode !== 0) { + output = `${stdout.trim()}\n\nSTDERR:\n${stderr.trim()}`; + } + } + + resolvePromise({ + tool: toolName, + exitCode, + output, + durationMs, + }); + }); + }); + } + + // ------------------------------------------------------------------------- + // Logging + // ------------------------------------------------------------------------- + + private logExecution(entry: ToolExecutionLogEntry): void { + this.config.log?.(entry); + } +} + +// --------------------------------------------------------------------------- +// Helpers: CLI arg building +// --------------------------------------------------------------------------- + +function buildCliArgs(toolArgs: Record): string[] { + let args: string[] = []; + + for (let [key, value] of Object.entries(toolArgs)) { + if (value === undefined || value === null) { + continue; + } + + if (typeof value === 'boolean') { + if (value) { + args.push(`--${key}`); + } + } else if (Array.isArray(value)) { + for (let item of value) { + args.push(`--${key}`, String(item)); + } + } else { + args.push(`--${key}`, String(value)); + } + } + + return args; +} + +function buildBoxelCliArgs( + _toolName: string, + subcommand: string, + toolArgs: Record, +): string[] { + let args: string[] = [subcommand]; + + // Certain tools use positional args rather than flags + switch (subcommand) { + case 'sync': { + let path = toolArgs['path']; + if (typeof path === 'string') { + args.push(path); + } + if (typeof toolArgs['prefer'] === 'string') { + args.push(`--prefer-${toolArgs['prefer']}`); + } + if (toolArgs['dry-run']) { + args.push('--dry-run'); + } + break; + } + case 'push': { + let localDir = toolArgs['local-dir']; + let realmUrl = toolArgs['realm-url']; + if (typeof localDir === 'string') { + args.push(localDir); + } + if (typeof realmUrl === 'string') { + args.push(realmUrl); + } + if (toolArgs['delete']) { + args.push('--delete'); + } + if (toolArgs['dry-run']) { + args.push('--dry-run'); + } + break; + } + case 'pull': { + let realmUrl = toolArgs['realm-url']; + let localDir = toolArgs['local-dir']; + if (typeof realmUrl === 'string') { + args.push(realmUrl); + } + if (typeof localDir === 'string') { + args.push(localDir); + } + if (toolArgs['delete']) { + args.push('--delete'); + } + if (toolArgs['dry-run']) { + args.push('--dry-run'); + } + break; + } + case 'status': { + let path = toolArgs['path']; + if (typeof path === 'string') { + args.push(path); + } + if (toolArgs['all']) { + args.push('--all'); + } + if (toolArgs['pull']) { + args.push('--pull'); + } + break; + } + case 'create': { + let endpoint = toolArgs['endpoint']; + let name = toolArgs['name']; + if (typeof endpoint === 'string') { + args.push(endpoint); + } + if (typeof name === 'string') { + args.push(name); + } + break; + } + case 'history': { + let path = toolArgs['path']; + if (typeof path === 'string') { + args.push(path); + } + if (typeof toolArgs['message'] === 'string') { + args.push('-m', toolArgs['message']); + } + break; + } + default: + // Fall through to generic flag building + args.push(...buildCliArgs(toolArgs)); + } + + return args; +} + +// --------------------------------------------------------------------------- +// Helpers: Realm API request building +// --------------------------------------------------------------------------- + +interface RealmApiRequestParams { + url: string; + method: string; + headers: Record; + body?: string; +} + +function buildRealmApiRequest( + toolName: string, + toolArgs: Record, + authorization?: string, +): RealmApiRequestParams { + let headers: Record = { + Accept: 'application/json', + }; + + if (authorization) { + headers['Authorization'] = authorization; + } + + switch (toolName) { + case 'realm-read': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let path = String(toolArgs['path']); + let accept = + typeof toolArgs['accept'] === 'string' + ? toolArgs['accept'] + : 'application/vnd.card+source'; + return { + url: `${realmUrl}${path}`, + method: 'GET', + headers: { ...headers, Accept: accept }, + }; + } + + case 'realm-write': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let path = String(toolArgs['path']); + let content = String(toolArgs['content']); + let contentType = + typeof toolArgs['content-type'] === 'string' + ? toolArgs['content-type'] + : 'application/vnd.card+source'; + return { + url: `${realmUrl}${path}`, + method: 'POST', + headers: { ...headers, 'Content-Type': contentType }, + body: content, + }; + } + + case 'realm-delete': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let path = String(toolArgs['path']); + return { + url: `${realmUrl}${path}`, + method: 'DELETE', + headers, + }; + } + + case 'realm-atomic': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let operations = String(toolArgs['operations']); + return { + url: `${realmUrl}_atomic`, + method: 'POST', + headers: { + ...headers, + 'Content-Type': 'application/vnd.api+json', + }, + body: JSON.stringify({ 'atomic:operations': JSON.parse(operations) }), + }; + } + + case 'realm-search': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + let query = String(toolArgs['query']); + return { + url: `${realmUrl}_search`, + method: 'QUERY', + headers: { + ...headers, + Accept: 'application/vnd.card+json', + 'Content-Type': 'application/json', + }, + body: query, + }; + } + + case 'realm-mtimes': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + return { + url: `${realmUrl}_mtimes`, + method: 'GET', + headers, + }; + } + + case 'realm-create': { + let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); + let name = String(toolArgs['name']); + return { + url: `${serverUrl}_create-realm`, + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + body: JSON.stringify({ name }), + }; + } + + case 'realm-server-session': { + let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); + return { + url: `${serverUrl}_server-session`, + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + body: '{}', + }; + } + + case 'realm-reindex': { + let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + return { + url: `${realmUrl}_reindex`, + method: 'POST', + headers, + }; + } + + default: + throw new Error(`Unknown realm-api tool: "${toolName}"`); + } +} + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- + +function ensureTrailingSlash(url: string): string { + return url.endsWith('/') ? url : `${url}/`; +} + +function looksLikeUrl(value: string): boolean { + return value.startsWith('http://') || value.startsWith('https://'); +} diff --git a/packages/software-factory/scripts/lib/factory-tool-registry.ts b/packages/software-factory/scripts/lib/factory-tool-registry.ts new file mode 100644 index 00000000000..0d5524c190d --- /dev/null +++ b/packages/software-factory/scripts/lib/factory-tool-registry.ts @@ -0,0 +1,595 @@ +import type { ToolArg, ToolManifest } from './factory-agent'; + +// --------------------------------------------------------------------------- +// Built-in tool manifests +// --------------------------------------------------------------------------- + +const SCRIPT_TOOLS: ToolManifest[] = [ + { + name: 'search-realm', + description: + 'Search for cards in a realm by type, field values, and sort criteria.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm', + type: 'string', + required: true, + description: 'Target realm URL', + }, + { + name: 'type-name', + type: 'string', + required: false, + description: 'Filter by card type name', + }, + { + name: 'type-module', + type: 'string', + required: false, + description: 'Filter by card type module', + }, + { + name: 'eq', + type: 'string', + required: false, + description: 'Equality filter as "field=value" (repeatable)', + }, + { + name: 'contains', + type: 'string', + required: false, + description: 'Contains filter as "field=value" (repeatable)', + }, + { + name: 'sort', + type: 'string', + required: false, + description: 'Sort as "field:direction" (repeatable)', + }, + { + name: 'size', + type: 'number', + required: false, + description: 'Page size', + }, + { + name: 'page', + type: 'number', + required: false, + description: 'Page number', + }, + ], + }, + { + name: 'pick-ticket', + description: 'Find tickets by status, priority, project, or agent.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm', + type: 'string', + required: true, + description: 'Target realm URL', + }, + { + name: 'status', + type: 'string', + required: false, + description: + 'Comma-separated status filter (default: backlog,in_progress,review)', + }, + { + name: 'project', + type: 'string', + required: false, + description: 'Filter by project ID', + }, + { + name: 'agent', + type: 'string', + required: false, + description: 'Filter by assigned agent ID', + }, + { + name: 'module', + type: 'string', + required: false, + description: 'Ticket schema module URL', + }, + ], + }, + { + name: 'get-session', + description: + 'Generate authenticated browser session tokens for realm access.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm', + type: 'string', + required: false, + description: 'Specific realm URL to include (repeatable)', + }, + ], + }, + { + name: 'run-realm-tests', + description: + 'Execute Playwright tests in an isolated scratch realm with fixture setup and teardown.', + category: 'script', + outputFormat: 'json', + args: [ + { + name: 'realm-path', + type: 'string', + required: false, + description: 'Source realm directory', + }, + { + name: 'realm-url', + type: 'string', + required: false, + description: 'Source realm URL', + }, + { + name: 'spec-dir', + type: 'string', + required: false, + description: 'Test directory (default: tests)', + }, + { + name: 'fixtures-dir', + type: 'string', + required: false, + description: 'Fixtures directory (default: tests/fixtures)', + }, + { + name: 'endpoint', + type: 'string', + required: false, + description: 'Realm endpoint name', + }, + { + name: 'scratch-root', + type: 'string', + required: false, + description: 'Base dir for test realms', + }, + ], + }, +]; + +const BOXEL_CLI_TOOLS: ToolManifest[] = [ + { + name: 'boxel-sync', + description: 'Bidirectional sync between local workspace and realm server.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'path', + type: 'string', + required: true, + description: 'Local workspace path', + }, + { + name: 'prefer', + type: 'string', + required: false, + description: 'Conflict strategy: "local", "remote", or "newest"', + }, + { + name: 'dry-run', + type: 'boolean', + required: false, + description: 'Preview only, no changes', + }, + ], + }, + { + name: 'boxel-push', + description: 'One-way upload from local directory to realm.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'local-dir', + type: 'string', + required: true, + description: 'Local directory path', + }, + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Target realm URL', + }, + { + name: 'delete', + type: 'boolean', + required: false, + description: 'Remove orphaned remote files', + }, + { + name: 'dry-run', + type: 'boolean', + required: false, + description: 'Preview only, no changes', + }, + ], + }, + { + name: 'boxel-pull', + description: 'One-way download from realm to local directory.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Source realm URL', + }, + { + name: 'local-dir', + type: 'string', + required: true, + description: 'Local directory path', + }, + { + name: 'delete', + type: 'boolean', + required: false, + description: 'Delete local files not on remote', + }, + { + name: 'dry-run', + type: 'boolean', + required: false, + description: 'Preview only, no changes', + }, + ], + }, + { + name: 'boxel-status', + description: 'Check sync status of a workspace.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'path', + type: 'string', + required: true, + description: 'Local workspace path', + }, + { + name: 'all', + type: 'boolean', + required: false, + description: 'Check all workspaces', + }, + { + name: 'pull', + type: 'boolean', + required: false, + description: 'Auto-pull remote changes', + }, + ], + }, + { + name: 'boxel-create', + description: 'Create a new workspace/realm endpoint.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'endpoint', + type: 'string', + required: true, + description: 'Endpoint type', + }, + { + name: 'name', + type: 'string', + required: true, + description: 'Workspace name', + }, + ], + }, + { + name: 'boxel-history', + description: 'View or create checkpoints for a workspace.', + category: 'boxel-cli', + outputFormat: 'text', + args: [ + { + name: 'path', + type: 'string', + required: true, + description: 'Local workspace path', + }, + { + name: 'message', + type: 'string', + required: false, + description: 'Checkpoint message', + }, + ], + }, +]; + +const REALM_API_TOOLS: ToolManifest[] = [ + { + name: 'realm-read', + description: 'Fetch a card or file from a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'path', + type: 'string', + required: true, + description: 'Card or file path within the realm', + }, + { + name: 'accept', + type: 'string', + required: false, + description: 'Accept header (default: application/vnd.card+source)', + }, + ], + }, + { + name: 'realm-write', + description: 'Create or update a card or file in a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'path', + type: 'string', + required: true, + description: 'Card or file path within the realm', + }, + { + name: 'content', + type: 'string', + required: true, + description: 'File content to write', + }, + { + name: 'content-type', + type: 'string', + required: false, + description: + 'Content-Type header (default: application/vnd.card+source)', + }, + ], + }, + { + name: 'realm-delete', + description: 'Delete a card or file from a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'path', + type: 'string', + required: true, + description: 'Card or file path to delete', + }, + ], + }, + { + name: 'realm-atomic', + description: 'Batch operations that succeed or fail atomically.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'operations', + type: 'string', + required: true, + description: + 'JSON array of operations: [{"op":"add|update|remove","href":"...","data":{...}}]', + }, + ], + }, + { + name: 'realm-search', + description: 'Search for cards using structured queries.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + { + name: 'query', + type: 'string', + required: true, + description: 'JSON search query object', + }, + ], + }, + { + name: 'realm-mtimes', + description: 'Get file modification times for a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm base URL', + }, + ], + }, + { + name: 'realm-create', + description: 'Create a new realm on the realm server.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-server-url', + type: 'string', + required: true, + description: 'Realm server base URL', + }, + { + name: 'name', + type: 'string', + required: true, + description: 'Name for the new realm', + }, + ], + }, + { + name: 'realm-server-session', + description: 'Obtain a realm server JWT for management operations.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-server-url', + type: 'string', + required: true, + description: 'Realm server base URL', + }, + ], + }, + { + name: 'realm-reindex', + description: 'Trigger a full reindex of a realm.', + category: 'realm-api', + outputFormat: 'json', + args: [ + { + name: 'realm-url', + type: 'string', + required: true, + description: 'Realm URL to reindex', + }, + ], + }, +]; + +// --------------------------------------------------------------------------- +// ToolRegistry +// --------------------------------------------------------------------------- + +export class ToolRegistry { + private manifestsByName: Map; + + constructor(manifests?: ToolManifest[]) { + let allManifests = manifests ?? [ + ...SCRIPT_TOOLS, + ...BOXEL_CLI_TOOLS, + ...REALM_API_TOOLS, + ]; + this.manifestsByName = new Map(allManifests.map((m) => [m.name, m])); + } + + /** Return all registered tool manifests. */ + getManifests(): ToolManifest[] { + return [...this.manifestsByName.values()]; + } + + /** Look up a single manifest by tool name. Returns undefined if not found. */ + getManifest(name: string): ToolManifest | undefined { + return this.manifestsByName.get(name); + } + + /** Check if a tool name is registered. */ + has(name: string): boolean { + return this.manifestsByName.has(name); + } + + /** Number of registered tools. */ + get size(): number { + return this.manifestsByName.size; + } + + /** + * Validate that a tool invocation's arguments satisfy the manifest. + * Returns an array of error messages (empty if valid). + */ + validateArgs( + toolName: string, + toolArgs: Record | undefined, + ): string[] { + let manifest = this.manifestsByName.get(toolName); + if (!manifest) { + return [`Unknown tool: "${toolName}"`]; + } + + let errors: string[] = []; + let requiredArgs = manifest.args.filter((a) => a.required); + + for (let arg of requiredArgs) { + let value = toolArgs?.[arg.name]; + if (value === undefined || value === null || value === '') { + errors.push( + `Missing required argument "${arg.name}" for tool "${toolName}"`, + ); + } + } + + return errors; + } +} + +// --------------------------------------------------------------------------- +// Convenience: default registry singleton +// --------------------------------------------------------------------------- + +let _defaultRegistry: ToolRegistry | undefined; + +export function getDefaultToolRegistry(): ToolRegistry { + if (!_defaultRegistry) { + _defaultRegistry = new ToolRegistry(); + } + return _defaultRegistry; +} + +// Re-export the built-in manifest arrays for testing +export { + SCRIPT_TOOLS, + BOXEL_CLI_TOOLS, + REALM_API_TOOLS, + type ToolArg, + type ToolManifest, +}; diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts new file mode 100644 index 00000000000..f5f006edbf7 --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -0,0 +1,681 @@ +import { module, test } from 'qunit'; + +import type { AgentAction, ToolResult } from '../scripts/lib/factory-agent'; +import { + ToolExecutor, + ToolNotFoundError, + ToolSafetyError, + ToolTimeoutError, + type ToolExecutionLogEntry, + type ToolExecutorConfig, +} from '../scripts/lib/factory-tool-executor'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeConfig( + overrides?: Partial, +): ToolExecutorConfig { + return { + packageRoot: '/fake/software-factory', + targetRealmUrl: 'https://realms.example.test/user/target/', + testRealmUrl: 'https://realms.example.test/user/target-tests/', + ...overrides, + }; +} + +function makeInvokeToolAction( + tool: string, + toolArgs?: Record, +): AgentAction { + return { + type: 'invoke_tool', + tool, + toolArgs, + }; +} + +// --------------------------------------------------------------------------- +// Unregistered tool rejection +// --------------------------------------------------------------------------- + +module('factory-tool-executor > unregistered tool rejection', function () { + test('rejects invoke_tool with empty tool name', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute({ type: 'invoke_tool' } as AgentAction); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolNotFoundError); + } + }); + + test('rejects unregistered tool', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute(makeInvokeToolAction('rm-rf-everything')); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolNotFoundError); + assert.true((err as Error).message.includes('rm-rf-everything')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Argument validation +// --------------------------------------------------------------------------- + +module('factory-tool-executor > argument validation', function () { + test('rejects missing required arguments', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute(makeInvokeToolAction('search-realm', {})); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof Error); + assert.true((err as Error).message.includes('realm')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Safety: source realm protection +// --------------------------------------------------------------------------- + +module('factory-tool-executor > source realm protection', function () { + test('rejects tool targeting source realm', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('search-realm', { + realm: 'https://realms.example.test/user/source/', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('source realm')); + } + }); + + test('rejects source realm without trailing slash', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('search-realm', { + realm: 'https://realms.example.test/user/source', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + } + }); + + test('allows tool targeting target realm', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + fetch: createMockFetch(200, { data: [] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/my-card.gts', + }), + ); + + assert.strictEqual(result.exitCode, 0); + }); + + test('allows tool targeting test realm', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + fetch: createMockFetch(200, { data: [] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target-tests/', + path: 'Test/spec.ts', + }), + ); + + assert.strictEqual(result.exitCode, 0); + }); + + test('rejects realm-api tool targeting unknown realm', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/other/unrelated/', + path: 'foo.json', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('not in the allowed list')); + } + }); + + test('allows realm-api tool targeting scratch realm via prefix', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + sourceRealmUrl: 'https://realms.example.test/user/source/', + allowedRealmPrefixes: ['https://realms.example.test/user/scratch-'], + fetch: createMockFetch(200, { ok: true }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/scratch-123/', + path: 'foo.json', + }), + ); + + assert.strictEqual(result.exitCode, 0); + }); +}); + +// --------------------------------------------------------------------------- +// Realm API execution +// --------------------------------------------------------------------------- + +module('factory-tool-executor > realm-api execution', function () { + test('realm-read makes GET request', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ card: 'data' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/my-card.gts', + }), + ); + + assert.strictEqual(capturedMethod, 'GET'); + assert.strictEqual( + capturedUrl, + 'https://realms.example.test/user/target/CardDef/my-card.gts', + ); + assert.strictEqual(result.exitCode, 0); + assert.deepEqual(result.output, { card: 'data' }); + assert.strictEqual(result.tool, 'realm-read'); + assert.strictEqual(typeof result.durationMs, 'number'); + }); + + test('realm-write makes POST request', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-write', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/new-card.gts', + content: 'export class NewCard {}', + }), + ); + + assert.strictEqual(capturedMethod, 'POST'); + assert.strictEqual( + capturedUrl, + 'https://realms.example.test/user/target/CardDef/new-card.gts', + ); + assert.strictEqual(capturedBody, 'export class NewCard {}'); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-delete makes DELETE request', async function (assert) { + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedMethod = init?.method; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-delete', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'CardDef/old-card.gts', + }), + ); + + assert.strictEqual(capturedMethod, 'DELETE'); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-search makes QUERY request', async function (assert) { + let capturedMethod: string | undefined; + let capturedUrl: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ data: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-search', { + 'realm-url': 'https://realms.example.test/user/target/', + query: JSON.stringify({ filter: { type: { name: 'Ticket' } } }), + }), + ); + + assert.strictEqual(capturedMethod, 'QUERY'); + assert.true(capturedUrl!.endsWith('_search')); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-atomic makes POST to _atomic endpoint', async function (assert) { + let capturedUrl: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let ops = [{ op: 'add', href: './Foo/bar.json', data: {} }]; + let result = await executor.execute( + makeInvokeToolAction('realm-atomic', { + 'realm-url': 'https://realms.example.test/user/target/', + operations: JSON.stringify(ops), + }), + ); + + assert.true(capturedUrl!.endsWith('_atomic')); + let body = JSON.parse(capturedBody!); + assert.deepEqual(body['atomic:operations'], ops); + assert.strictEqual(result.exitCode, 0); + }); + + test('non-ok response produces exitCode 1', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(404, { error: 'Not found' }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'missing.json', + }), + ); + + assert.strictEqual(result.exitCode, 1); + assert.deepEqual( + (result.output as Record).error, + 'HTTP 404', + ); + }); + + test('includes authorization header when configured', async function (assert) { + let capturedHeaders: Headers | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + authorization: 'Bearer test-token-123', + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = new Headers(init?.headers as HeadersInit); + return new Response(JSON.stringify({}), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + assert.strictEqual( + capturedHeaders!.get('Authorization'), + 'Bearer test-token-123', + ); + }); + + test('realm-mtimes makes GET to _mtimes', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ 'foo.json': 12345 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-mtimes', { + 'realm-url': 'https://realms.example.test/user/target/', + }), + ); + + assert.strictEqual(capturedMethod, 'GET'); + assert.true(capturedUrl!.endsWith('_mtimes')); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-create makes POST to _create-realm', async function (assert) { + let capturedUrl: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/user/target/', + name: 'my-scratch-realm', + }), + ); + + assert.true(capturedUrl!.endsWith('_create-realm')); + let body = JSON.parse(capturedBody!); + assert.strictEqual(body.name, 'my-scratch-realm'); + assert.strictEqual(result.exitCode, 0); + }); + + test('realm-reindex makes POST to _reindex', async function (assert) { + let capturedUrl: string | undefined; + let capturedMethod: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedMethod = init?.method; + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-reindex', { + 'realm-url': 'https://realms.example.test/user/target/', + }), + ); + + assert.strictEqual(capturedMethod, 'POST'); + assert.true(capturedUrl!.endsWith('_reindex')); + assert.strictEqual(result.exitCode, 0); + }); +}); + +// --------------------------------------------------------------------------- +// Logging +// --------------------------------------------------------------------------- + +module('factory-tool-executor > logging', function () { + test('logs successful tool execution', async function (assert) { + let logEntries: ToolExecutionLogEntry[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(200, { data: [] }), + log: (entry) => logEntries.push(entry), + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + assert.strictEqual(logEntries.length, 1); + assert.strictEqual(logEntries[0].tool, 'realm-read'); + assert.strictEqual(logEntries[0].category, 'realm-api'); + assert.strictEqual(logEntries[0].exitCode, 0); + assert.strictEqual(typeof logEntries[0].durationMs, 'number'); + assert.strictEqual(logEntries[0].error, undefined); + }); + + test('logs failed tool execution', async function (assert) { + let logEntries: ToolExecutionLogEntry[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(500, { error: 'Internal error' }), + log: (entry) => logEntries.push(entry), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'broken.json', + }), + ); + + assert.strictEqual(result.exitCode, 1); + assert.strictEqual(logEntries.length, 1); + assert.strictEqual(logEntries[0].exitCode, 1); + }); +}); + +// --------------------------------------------------------------------------- +// ToolResult serialization +// --------------------------------------------------------------------------- + +module('factory-tool-executor > ToolResult shape', function () { + test('successful result has expected shape', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(200, { cards: ['a', 'b'] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + assert.strictEqual(typeof result.tool, 'string'); + assert.strictEqual(typeof result.exitCode, 'number'); + assert.strictEqual(typeof result.durationMs, 'number'); + assert.notStrictEqual(result.output, undefined, 'output is defined'); + }); + + test('ToolResult can be serialized to JSON', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: createMockFetch(200, { data: [1, 2, 3] }), + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'foo.json', + }), + ); + + let serialized = JSON.stringify(result); + let deserialized = JSON.parse(serialized) as ToolResult; + assert.strictEqual(deserialized.tool, 'realm-read'); + assert.strictEqual(deserialized.exitCode, 0); + assert.deepEqual(deserialized.output, { data: [1, 2, 3] }); + }); +}); + +// --------------------------------------------------------------------------- +// Timeout behavior +// --------------------------------------------------------------------------- + +module('factory-tool-executor > timeout', function () { + test('realm-api call times out with ToolTimeoutError', async function (assert) { + let registry = new ToolRegistry(); + let config = makeConfig({ + timeoutMs: 50, + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + // Respect the AbortSignal so the timeout mechanism works + return new Promise((resolve, reject) => { + let signal = init?.signal; + if (signal) { + signal.addEventListener('abort', () => { + reject( + new DOMException('The operation was aborted.', 'AbortError'), + ); + }); + } + // Never resolves on its own within timeout + setTimeout( + () => + resolve( + new Response('{}', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ), + 5000, + ); + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + try { + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'slow.json', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolTimeoutError); + assert.true((err as Error).message.includes('50ms')); + } + }); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function createMockFetch( + status: number, + body: unknown, +): typeof globalThis.fetch { + return (async () => { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} diff --git a/packages/software-factory/tests/factory-tool-registry.test.ts b/packages/software-factory/tests/factory-tool-registry.test.ts new file mode 100644 index 00000000000..b616661e07f --- /dev/null +++ b/packages/software-factory/tests/factory-tool-registry.test.ts @@ -0,0 +1,281 @@ +import { module, test } from 'qunit'; + +import { + BOXEL_CLI_TOOLS, + getDefaultToolRegistry, + REALM_API_TOOLS, + SCRIPT_TOOLS, + ToolRegistry, + type ToolManifest, +} from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// ToolRegistry construction +// --------------------------------------------------------------------------- + +module('factory-tool-registry > ToolRegistry construction', function () { + test('default registry includes all built-in tools', function (assert) { + let registry = new ToolRegistry(); + let expectedCount = + SCRIPT_TOOLS.length + BOXEL_CLI_TOOLS.length + REALM_API_TOOLS.length; + assert.strictEqual( + registry.size, + expectedCount, + `registry has ${expectedCount} tools`, + ); + }); + + test('accepts custom manifest list', function (assert) { + let custom: ToolManifest[] = [ + { + name: 'custom-tool', + description: 'A test tool', + category: 'script', + args: [], + outputFormat: 'json', + }, + ]; + let registry = new ToolRegistry(custom); + assert.strictEqual(registry.size, 1); + assert.true(registry.has('custom-tool')); + }); + + test('empty manifest list creates empty registry', function (assert) { + let registry = new ToolRegistry([]); + assert.strictEqual(registry.size, 0); + }); +}); + +// --------------------------------------------------------------------------- +// getManifests +// --------------------------------------------------------------------------- + +module('factory-tool-registry > getManifests', function () { + test('returns all manifests', function (assert) { + let registry = new ToolRegistry(); + let manifests = registry.getManifests(); + assert.true(manifests.length > 0, 'returns non-empty array'); + assert.true(Array.isArray(manifests), 'returns an array'); + }); + + test('returned array is a copy (not internal state)', function (assert) { + let registry = new ToolRegistry(); + let a = registry.getManifests(); + let b = registry.getManifests(); + assert.notStrictEqual(a, b, 'different array instances'); + assert.deepEqual(a, b, 'same contents'); + }); +}); + +// --------------------------------------------------------------------------- +// getManifest +// --------------------------------------------------------------------------- + +module('factory-tool-registry > getManifest', function () { + test('returns manifest for known tool', function (assert) { + let registry = new ToolRegistry(); + let manifest = registry.getManifest('search-realm'); + assert.ok(manifest, 'manifest is defined'); + assert.strictEqual(manifest!.name, 'search-realm'); + assert.strictEqual(manifest!.category, 'script'); + }); + + test('returns undefined for unknown tool', function (assert) { + let registry = new ToolRegistry(); + assert.strictEqual(registry.getManifest('nonexistent'), undefined); + }); +}); + +// --------------------------------------------------------------------------- +// has +// --------------------------------------------------------------------------- + +module('factory-tool-registry > has', function () { + test('returns true for registered tool', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('search-realm')); + assert.true(registry.has('boxel-sync')); + assert.true(registry.has('realm-read')); + }); + + test('returns false for unregistered tool', function (assert) { + let registry = new ToolRegistry(); + assert.false(registry.has('rm-rf')); + assert.false(registry.has('')); + }); +}); + +// --------------------------------------------------------------------------- +// validateArgs +// --------------------------------------------------------------------------- + +module('factory-tool-registry > validateArgs', function () { + test('returns empty array for valid args', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { + realm: 'http://example.test/', + }); + assert.deepEqual(errors, []); + }); + + test('returns error for missing required arg', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', {}); + assert.strictEqual(errors.length, 1); + assert.true(errors[0].includes('realm')); + }); + + test('returns error for unknown tool', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('nonexistent', {}); + assert.strictEqual(errors.length, 1); + assert.true(errors[0].includes('Unknown tool')); + }); + + test('multiple missing required args produce multiple errors', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('realm-write', {}); + assert.true( + errors.length >= 3, + `expected at least 3 errors, got ${errors.length}`, + ); + }); + + test('optional args do not produce errors when missing', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { + realm: 'http://example.test/', + }); + assert.deepEqual(errors, [], 'no errors for missing optional args'); + }); + + test('empty string for required arg produces error', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { realm: '' }); + assert.strictEqual(errors.length, 1); + }); +}); + +// --------------------------------------------------------------------------- +// Built-in manifest completeness +// --------------------------------------------------------------------------- + +module('factory-tool-registry > built-in manifests', function () { + test('all script tools have correct category', function (assert) { + for (let tool of SCRIPT_TOOLS) { + assert.strictEqual( + tool.category, + 'script', + `${tool.name} has category "script"`, + ); + } + }); + + test('all boxel-cli tools have correct category', function (assert) { + for (let tool of BOXEL_CLI_TOOLS) { + assert.strictEqual( + tool.category, + 'boxel-cli', + `${tool.name} has category "boxel-cli"`, + ); + } + }); + + test('all realm-api tools have correct category', function (assert) { + for (let tool of REALM_API_TOOLS) { + assert.strictEqual( + tool.category, + 'realm-api', + `${tool.name} has category "realm-api"`, + ); + } + }); + + test('all tools have unique names', function (assert) { + let registry = new ToolRegistry(); + let manifests = registry.getManifests(); + let names = manifests.map((m) => m.name); + let unique = new Set(names); + assert.strictEqual(unique.size, names.length, 'all tool names are unique'); + }); + + test('all tools have non-empty description', function (assert) { + let registry = new ToolRegistry(); + for (let manifest of registry.getManifests()) { + assert.true( + manifest.description.length > 0, + `${manifest.name} has a description`, + ); + } + }); + + test('all tool args have required fields', function (assert) { + let registry = new ToolRegistry(); + for (let manifest of registry.getManifests()) { + for (let arg of manifest.args) { + assert.true(arg.name.length > 0, `${manifest.name} arg has a name`); + assert.true( + arg.type.length > 0, + `${manifest.name}:${arg.name} has a type`, + ); + assert.true( + arg.description.length > 0, + `${manifest.name}:${arg.name} has a description`, + ); + assert.strictEqual( + typeof arg.required, + 'boolean', + `${manifest.name}:${arg.name} has required flag`, + ); + } + } + }); + + test('expected script tools are registered', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('search-realm')); + assert.true(registry.has('pick-ticket')); + assert.true(registry.has('get-session')); + assert.true(registry.has('run-realm-tests')); + }); + + test('expected boxel-cli tools are registered', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('boxel-sync')); + assert.true(registry.has('boxel-push')); + assert.true(registry.has('boxel-pull')); + assert.true(registry.has('boxel-status')); + assert.true(registry.has('boxel-create')); + assert.true(registry.has('boxel-history')); + }); + + test('expected realm-api tools are registered', function (assert) { + let registry = new ToolRegistry(); + assert.true(registry.has('realm-read')); + assert.true(registry.has('realm-write')); + assert.true(registry.has('realm-delete')); + assert.true(registry.has('realm-atomic')); + assert.true(registry.has('realm-search')); + assert.true(registry.has('realm-mtimes')); + assert.true(registry.has('realm-create')); + assert.true(registry.has('realm-server-session')); + assert.true(registry.has('realm-reindex')); + }); +}); + +// --------------------------------------------------------------------------- +// getDefaultToolRegistry +// --------------------------------------------------------------------------- + +module('factory-tool-registry > getDefaultToolRegistry', function () { + test('returns a ToolRegistry instance', function (assert) { + let registry = getDefaultToolRegistry(); + assert.true(registry instanceof ToolRegistry); + }); + + test('returns the same instance on repeated calls', function (assert) { + let a = getDefaultToolRegistry(); + let b = getDefaultToolRegistry(); + assert.strictEqual(a, b, 'same singleton instance'); + }); +}); diff --git a/packages/software-factory/tests/index.ts b/packages/software-factory/tests/index.ts index e34a4421137..4201f58613a 100644 --- a/packages/software-factory/tests/index.ts +++ b/packages/software-factory/tests/index.ts @@ -4,4 +4,6 @@ import './factory-brief.test'; import './factory-entrypoint.test'; import './factory-entrypoint.integration.test'; import './factory-target-realm.test'; +import './factory-tool-executor.test'; +import './factory-tool-registry.test'; import './realm-auth.test'; From 740e9de6a61cf3f79ab99fcc39e13e3ac03e3a3a Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 17:31:17 -0400 Subject: [PATCH 02/13] Address PR review feedback: realm API contracts, safety, and validation - realm-create: send JSON:API body with data.type/attributes.endpoint/name - realm-server-session: send OpenID token, capture Authorization header JWT - enforceRealmSafety: always validate realm-api targets even without sourceRealmUrl - ToolRegistry constructor: detect and throw on duplicate tool names - validateArgs: reject whitespace-only strings for required args - executeRealmApi: handle empty JSON response bodies gracefully Co-Authored-By: Claude Opus 4.6 (1M context) --- .../scripts/lib/factory-tool-executor.ts | 81 +++++++++++------- .../scripts/lib/factory-tool-registry.ts | 36 +++++++- .../tests/factory-tool-executor.test.ts | 83 ++++++++++++++++++- .../tests/factory-tool-registry.test.ts | 30 +++++++ 4 files changed, 193 insertions(+), 37 deletions(-) diff --git a/packages/software-factory/scripts/lib/factory-tool-executor.ts b/packages/software-factory/scripts/lib/factory-tool-executor.ts index 3a7c7b02300..c99866ab58a 100644 --- a/packages/software-factory/scripts/lib/factory-tool-executor.ts +++ b/packages/software-factory/scripts/lib/factory-tool-executor.ts @@ -207,35 +207,33 @@ export class ToolExecutor { toolName: string, toolArgs: Record, ): void { + // Source realm protection (when configured) let sourceUrl = this.config.sourceRealmUrl; - if (!sourceUrl) { - return; - } - - let normalizedSource = ensureTrailingSlash(sourceUrl); - - // Check all string args that look like realm URLs - let realmArgNames = [ - 'realm', - 'realm-url', - 'realm-server-url', - 'local-dir', - 'path', - ]; - - for (let argName of realmArgNames) { - let value = toolArgs[argName]; - if (typeof value === 'string' && looksLikeUrl(value)) { - let normalizedValue = ensureTrailingSlash(value); - if (normalizedValue === normalizedSource) { - throw new ToolSafetyError( - `Tool "${toolName}" cannot target the source realm: ${sourceUrl}`, - ); + if (sourceUrl) { + let normalizedSource = ensureTrailingSlash(sourceUrl); + + let realmArgNames = [ + 'realm', + 'realm-url', + 'realm-server-url', + 'local-dir', + 'path', + ]; + + for (let argName of realmArgNames) { + let value = toolArgs[argName]; + if (typeof value === 'string' && looksLikeUrl(value)) { + let normalizedValue = ensureTrailingSlash(value); + if (normalizedValue === normalizedSource) { + throw new ToolSafetyError( + `Tool "${toolName}" cannot target the source realm: ${sourceUrl}`, + ); + } } } } - // For realm-api tools, also check if the target URL is allowed + // Allowed-realm targeting for realm-api tools (always enforced) let manifest = this.registry.getManifest(toolName); if (manifest?.category === 'realm-api') { let realmUrl = toolArgs['realm-url']; @@ -372,10 +370,22 @@ export class ToolExecutor { let responseBody: unknown; let contentType = response.headers.get('content-type') ?? ''; - if (contentType.includes('json')) { - responseBody = await response.json(); + let rawText = await response.text(); + if (contentType.includes('json') && rawText.length > 0) { + try { + responseBody = JSON.parse(rawText); + } catch { + responseBody = rawText; + } } else { - responseBody = await response.text(); + responseBody = rawText; + } + + // Some endpoints return important values in headers (e.g. _server-session + // returns the JWT in the Authorization header with an empty/null body). + let authorizationHeader = response.headers.get('authorization'); + if (toolName === 'realm-server-session' && authorizationHeader) { + responseBody = { token: authorizationHeader }; } return { @@ -717,21 +727,32 @@ function buildRealmApiRequest( case 'realm-create': { let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); let name = String(toolArgs['name']); + let endpoint = String(toolArgs['endpoint']); return { url: `${serverUrl}_create-realm`, method: 'POST', - headers: { ...headers, 'Content-Type': 'application/json' }, - body: JSON.stringify({ name }), + headers: { + ...headers, + Accept: 'application/vnd.api+json', + 'Content-Type': 'application/vnd.api+json', + }, + body: JSON.stringify({ + data: { + type: 'realm', + attributes: { name, endpoint }, + }, + }), }; } case 'realm-server-session': { let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); + let openidToken = String(toolArgs['openid-token']); return { url: `${serverUrl}_server-session`, method: 'POST', headers: { ...headers, 'Content-Type': 'application/json' }, - body: '{}', + body: JSON.stringify({ access_token: openidToken }), }; } diff --git a/packages/software-factory/scripts/lib/factory-tool-registry.ts b/packages/software-factory/scripts/lib/factory-tool-registry.ts index 0d5524c190d..3efd93622bb 100644 --- a/packages/software-factory/scripts/lib/factory-tool-registry.ts +++ b/packages/software-factory/scripts/lib/factory-tool-registry.ts @@ -473,13 +473,21 @@ const REALM_API_TOOLS: ToolManifest[] = [ name: 'name', type: 'string', required: true, - description: 'Name for the new realm', + description: 'Display name for the new realm', + }, + { + name: 'endpoint', + type: 'string', + required: true, + description: + 'URL path segment for the new realm (e.g. "user/my-realm")', }, ], }, { name: 'realm-server-session', - description: 'Obtain a realm server JWT for management operations.', + description: + 'Obtain a realm server JWT for management operations. Returns the JWT in the output.', category: 'realm-api', outputFormat: 'json', args: [ @@ -489,6 +497,13 @@ const REALM_API_TOOLS: ToolManifest[] = [ required: true, description: 'Realm server base URL', }, + { + name: 'openid-token', + type: 'string', + required: true, + description: + 'OpenID access_token obtained from the Matrix server via /openid/request_token', + }, ], }, { @@ -520,7 +535,16 @@ export class ToolRegistry { ...BOXEL_CLI_TOOLS, ...REALM_API_TOOLS, ]; - this.manifestsByName = new Map(allManifests.map((m) => [m.name, m])); + let map = new Map(); + for (let manifest of allManifests) { + if (map.has(manifest.name)) { + throw new Error( + `Duplicate tool manifest name "${manifest.name}" detected in ToolRegistry`, + ); + } + map.set(manifest.name, manifest); + } + this.manifestsByName = map; } /** Return all registered tool manifests. */ @@ -561,7 +585,11 @@ export class ToolRegistry { for (let arg of requiredArgs) { let value = toolArgs?.[arg.name]; - if (value === undefined || value === null || value === '') { + let isEmpty = + value === undefined || + value === null || + (typeof value === 'string' && value.trim() === ''); + if (isEmpty) { errors.push( `Missing required argument "${arg.name}" for tool "${toolName}"`, ); diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts index f5f006edbf7..5f6f4fc279b 100644 --- a/packages/software-factory/tests/factory-tool-executor.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -212,6 +212,29 @@ module('factory-tool-executor > source realm protection', function () { assert.strictEqual(result.exitCode, 0); }); + + test('rejects unknown realm even without sourceRealmUrl configured', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + // No sourceRealmUrl — safety should still enforce allowed-realm targeting + }), + ); + + try { + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://evil.example.test/hacker/realm/', + path: 'secrets.json', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('not in the allowed list')); + } + }); }); // --------------------------------------------------------------------------- @@ -473,14 +496,26 @@ module('factory-tool-executor > realm-api execution', function () { let result = await executor.execute( makeInvokeToolAction('realm-create', { - 'realm-server-url': 'https://realms.example.test/user/target/', + 'realm-server-url': 'https://realms.example.test/', name: 'my-scratch-realm', + endpoint: 'user/scratch-123', }), ); - assert.true(capturedUrl!.endsWith('_create-realm')); + assert.strictEqual( + capturedUrl, + 'https://realms.example.test/_create-realm', + ); let body = JSON.parse(capturedBody!); - assert.strictEqual(body.name, 'my-scratch-realm'); + assert.deepEqual(body, { + data: { + type: 'realm', + attributes: { + name: 'my-scratch-realm', + endpoint: 'user/scratch-123', + }, + }, + }); assert.strictEqual(result.exitCode, 0); }); @@ -511,6 +546,48 @@ module('factory-tool-executor > realm-api execution', function () { assert.true(capturedUrl!.endsWith('_reindex')); assert.strictEqual(result.exitCode, 0); }); + + test('realm-server-session sends OpenID token and captures Authorization header', async function (assert) { + let capturedUrl: string | undefined; + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + capturedUrl = String(input); + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response(null, { + status: 201, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer realm-server-jwt-123', + }, + }); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + let result = await executor.execute( + makeInvokeToolAction('realm-server-session', { + 'realm-server-url': 'https://realms.example.test/user/target/', + 'openid-token': 'openid-access-token-xyz', + }), + ); + + assert.true(capturedUrl!.endsWith('_server-session')); + let body = JSON.parse(capturedBody!); + assert.strictEqual( + body.access_token, + 'openid-access-token-xyz', + 'sends OpenID token in request body', + ); + assert.strictEqual(result.exitCode, 0); + assert.deepEqual( + result.output, + { token: 'Bearer realm-server-jwt-123' }, + 'captures Authorization header in output', + ); + }); }); // --------------------------------------------------------------------------- diff --git a/packages/software-factory/tests/factory-tool-registry.test.ts b/packages/software-factory/tests/factory-tool-registry.test.ts index b616661e07f..6bcfb5fa7b7 100644 --- a/packages/software-factory/tests/factory-tool-registry.test.ts +++ b/packages/software-factory/tests/factory-tool-registry.test.ts @@ -44,6 +44,30 @@ module('factory-tool-registry > ToolRegistry construction', function () { let registry = new ToolRegistry([]); assert.strictEqual(registry.size, 0); }); + + test('throws on duplicate tool names', function (assert) { + let dupes: ToolManifest[] = [ + { + name: 'same-name', + description: 'first', + category: 'script', + args: [], + outputFormat: 'json', + }, + { + name: 'same-name', + description: 'second', + category: 'script', + args: [], + outputFormat: 'json', + }, + ]; + assert.throws( + () => new ToolRegistry(dupes), + (err: Error) => + err.message.includes('Duplicate') && err.message.includes('same-name'), + ); + }); }); // --------------------------------------------------------------------------- @@ -154,6 +178,12 @@ module('factory-tool-registry > validateArgs', function () { let errors = registry.validateArgs('search-realm', { realm: '' }); assert.strictEqual(errors.length, 1); }); + + test('whitespace-only string for required arg produces error', function (assert) { + let registry = new ToolRegistry(); + let errors = registry.validateArgs('search-realm', { realm: ' ' }); + assert.strictEqual(errors.length, 1, 'whitespace-only value is rejected'); + }); }); // --------------------------------------------------------------------------- From 34e21ed5cf7fb4ec655b14d07a985fb11a9425ce Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 17:33:54 -0400 Subject: [PATCH 03/13] Add auth header propagation tests for all realm-api tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verify that the realm JWT / realm-server JWT is correctly sent as the Authorization header for every realm-api tool (read, write, delete, search, atomic, create, reindex). Also tests the end-to-end flow: realm-server-session → mint JWT → use JWT in realm-create. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../tests/factory-tool-executor.test.ts | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts index 5f6f4fc279b..6dcfffdf7aa 100644 --- a/packages/software-factory/tests/factory-tool-executor.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -590,6 +590,262 @@ module('factory-tool-executor > realm-api execution', function () { }); }); +// --------------------------------------------------------------------------- +// Auth header propagation +// --------------------------------------------------------------------------- + +module('factory-tool-executor > auth header propagation', function () { + function createHeaderCapturingFetch(): { + fetch: typeof globalThis.fetch; + getCapturedHeaders: () => Headers | undefined; + } { + let capturedHeaders: Headers | undefined; + return { + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = new Headers(init?.headers as HeadersInit); + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + getCapturedHeaders: () => capturedHeaders, + }; + } + + test('realm-read sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/foo.json', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-write sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-write', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/new.gts', + content: 'export class NewCard {}', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-delete sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-delete', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/old.json', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-search sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-search', { + 'realm-url': 'https://realms.example.test/user/target/', + query: '{}', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-atomic sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-atomic', { + 'realm-url': 'https://realms.example.test/user/target/', + operations: '[]', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('realm-create sends realm-server JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ + authorization: 'Bearer realm-server-jwt-xyz', + fetch, + }), + ); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/user/target/', + name: 'scratch', + endpoint: 'user/scratch', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-server-jwt-xyz', + ); + }); + + test('realm-reindex sends realm JWT in Authorization header', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ authorization: 'Bearer realm-jwt-abc', fetch }), + ); + + await executor.execute( + makeInvokeToolAction('realm-reindex', { + 'realm-url': 'https://realms.example.test/user/target/', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + 'Bearer realm-jwt-abc', + ); + }); + + test('no Authorization header when authorization is not configured', async function (assert) { + let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeConfig({ fetch }), // no authorization + ); + + await executor.execute( + makeInvokeToolAction('realm-read', { + 'realm-url': 'https://realms.example.test/user/target/', + path: 'Card/foo.json', + }), + ); + + assert.strictEqual( + getCapturedHeaders()!.get('Authorization'), + null, + 'no Authorization header sent', + ); + }); + + test('realm-server-session JWT can be used for subsequent realm-create', async function (assert) { + let registry = new ToolRegistry(); + + // Step 1: Get realm server JWT via realm-server-session + let sessionExecutor = new ToolExecutor( + registry, + makeConfig({ + fetch: (async () => { + return new Response(null, { + status: 201, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer minted-realm-server-jwt', + }, + }); + }) as typeof globalThis.fetch, + }), + ); + + let sessionResult = await sessionExecutor.execute( + makeInvokeToolAction('realm-server-session', { + 'realm-server-url': 'https://realms.example.test/user/target/', + 'openid-token': 'matrix-openid-token', + }), + ); + + let jwt = (sessionResult.output as { token: string }).token; + assert.strictEqual(jwt, 'Bearer minted-realm-server-jwt'); + + // Step 2: Use the JWT for realm-create + let capturedHeaders: Headers | undefined; + let createExecutor = new ToolExecutor( + registry, + makeConfig({ + authorization: jwt, + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = new Headers(init?.headers as HeadersInit); + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch, + }), + ); + + await createExecutor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/user/target/', + name: 'test-realm', + endpoint: 'user/test-realm', + }), + ); + + assert.strictEqual( + capturedHeaders!.get('Authorization'), + 'Bearer minted-realm-server-jwt', + 'realm-create uses the JWT from realm-server-session', + ); + }); +}); + // --------------------------------------------------------------------------- // Logging // --------------------------------------------------------------------------- From cf1a37953cde6c2a2f90cffae74cd4f0e3fe3846 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 17:51:29 -0400 Subject: [PATCH 04/13] Add integration tests: realm-api request building and safety constraints Integration tests spin up a real HTTP server and verify the executor sends correct URLs, methods, headers, and bodies for all 9 realm-api tools. Also verifies safety constraints reject before any HTTP request reaches the server (unregistered tools, source realm, unknown realm). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../factory-tool-executor.integration.test.ts | 680 ++++++++++++++++++ packages/software-factory/tests/index.ts | 1 + 2 files changed, 681 insertions(+) create mode 100644 packages/software-factory/tests/factory-tool-executor.integration.test.ts diff --git a/packages/software-factory/tests/factory-tool-executor.integration.test.ts b/packages/software-factory/tests/factory-tool-executor.integration.test.ts new file mode 100644 index 00000000000..d48679b7c93 --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.integration.test.ts @@ -0,0 +1,680 @@ +import { createServer, type IncomingMessage, type Server } from 'node:http'; +import { module, test } from 'qunit'; + +import { ToolExecutor } from '../scripts/lib/factory-tool-executor'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Test server helpers +// --------------------------------------------------------------------------- + +interface CapturedRequest { + method: string; + url: string; + headers: Record; + body: string; +} + +function startTestServer( + handler: ( + req: CapturedRequest, + respond: ( + status: number, + body: unknown, + headers?: Record, + ) => void, + ) => void, +): Promise<{ server: Server; origin: string }> { + return new Promise((resolve) => { + let server = createServer(async (req: IncomingMessage, res) => { + let body = ''; + for await (let chunk of req) { + body += chunk; + } + + let captured: CapturedRequest = { + method: req.method ?? 'GET', + url: req.url ?? '/', + headers: req.headers, + body, + }; + + handler(captured, (status, responseBody, headers) => { + res.writeHead(status, { + 'Content-Type': 'application/json', + ...headers, + }); + res.end( + responseBody !== null && responseBody !== undefined + ? JSON.stringify(responseBody) + : '', + ); + }); + }); + + server.listen(0, () => { + let address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected test server to bind to a TCP port'); + } + let origin = `http://127.0.0.1:${address.port}`; + resolve({ server, origin }); + }); + }); +} + +function stopServer(server: Server): Promise { + return new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())), + ); +} + +// --------------------------------------------------------------------------- +// Integration tests: realm-api tools against a real HTTP server +// --------------------------------------------------------------------------- + +module('factory-tool-executor integration > realm-api requests', function () { + test('realm-read sends correct GET with Authorization and Accept headers', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { data: { id: 'Card/hello', type: 'card' } }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': realmUrl, + path: 'Card/hello.gts', + }, + }); + + assert.strictEqual(result.exitCode, 0, 'exitCode is 0'); + assert.strictEqual(captured!.method, 'GET'); + assert.strictEqual(captured!.url, '/user/target/Card/hello.gts'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + assert.strictEqual( + captured!.headers.accept, + 'application/vnd.card+source', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-write sends correct POST with content and headers', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { ok: true }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-write', + toolArgs: { + 'realm-url': realmUrl, + path: 'CardDef/my-card.gts', + content: 'export class MyCard extends CardDef {}', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/CardDef/my-card.gts'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + assert.strictEqual( + captured!.headers['content-type'], + 'application/vnd.card+source', + ); + assert.strictEqual( + captured!.body, + 'export class MyCard extends CardDef {}', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-delete sends correct DELETE with Authorization header', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(204, null); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-delete', + toolArgs: { + 'realm-url': realmUrl, + path: 'Card/old-card.json', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'DELETE'); + assert.strictEqual(captured!.url, '/user/target/Card/old-card.json'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-search sends correct QUERY to _search with JSON body', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { data: [] }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let query = JSON.stringify({ + filter: { + type: { module: 'https://example.test/ticket', name: 'Ticket' }, + }, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': realmUrl, + query, + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'QUERY'); + assert.strictEqual(captured!.url, '/user/target/_search'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + assert.strictEqual(captured!.headers.accept, 'application/vnd.card+json'); + assert.strictEqual(captured!.headers['content-type'], 'application/json'); + assert.strictEqual(captured!.body, query); + } finally { + await stopServer(server); + } + }); + + test('realm-atomic sends correct POST to _atomic with JSON:API operations', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { ok: true }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let ops = [ + { op: 'add', href: './CardDef/new.gts', data: { type: 'module' } }, + { op: 'remove', href: './Card/old.json' }, + ]; + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-atomic', + toolArgs: { + 'realm-url': realmUrl, + operations: JSON.stringify(ops), + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_atomic'); + assert.strictEqual( + captured!.headers['content-type'], + 'application/vnd.api+json', + ); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + + let body = JSON.parse(captured!.body); + assert.deepEqual(body['atomic:operations'], ops); + } finally { + await stopServer(server); + } + }); + + test('realm-mtimes sends correct GET to _mtimes', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { 'Card/foo.json': 1700000000 }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-mtimes', + toolArgs: { 'realm-url': realmUrl }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'GET'); + assert.strictEqual(captured!.url, '/user/target/_mtimes'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-create sends correct JSON:API POST to _create-realm with realm-server JWT', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(201, { + data: { type: 'realm', id: `${origin}/user/new-realm/` }, + }); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-server-jwt-minted', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-create', + toolArgs: { + 'realm-server-url': `${origin}/user/target/`, + name: 'New Realm', + endpoint: 'user/new-realm', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_create-realm'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-server-jwt-minted', + ); + assert.strictEqual(captured!.headers.accept, 'application/vnd.api+json'); + assert.strictEqual( + captured!.headers['content-type'], + 'application/vnd.api+json', + ); + + let body = JSON.parse(captured!.body); + assert.deepEqual(body, { + data: { + type: 'realm', + attributes: { + name: 'New Realm', + endpoint: 'user/new-realm', + }, + }, + }); + } finally { + await stopServer(server); + } + }); + + test('realm-server-session sends OpenID token and returns JWT from Authorization header', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(201, null, { + Authorization: 'Bearer freshly-minted-jwt', + }); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-server-session', + toolArgs: { + 'realm-server-url': `${origin}/user/target/`, + 'openid-token': 'matrix-openid-access-token', + }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_server-session'); + assert.strictEqual(captured!.headers['content-type'], 'application/json'); + + let body = JSON.parse(captured!.body); + assert.strictEqual( + body.access_token, + 'matrix-openid-access-token', + 'sends OpenID token in request body', + ); + + assert.deepEqual( + result.output, + { token: 'Bearer freshly-minted-jwt' }, + 'captures JWT from Authorization response header', + ); + } finally { + await stopServer(server); + } + }); + + test('realm-reindex sends correct POST to _reindex', async function (assert) { + let captured: CapturedRequest | undefined; + + let { server, origin } = await startTestServer((req, respond) => { + captured = req; + respond(200, { status: 'reindexing' }); + }); + + try { + let registry = new ToolRegistry(); + let realmUrl = `${origin}/user/target/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: realmUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: 'Bearer realm-jwt-for-user', + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-reindex', + toolArgs: { 'realm-url': realmUrl }, + }); + + assert.strictEqual(result.exitCode, 0); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_reindex'); + assert.strictEqual( + captured!.headers.authorization, + 'Bearer realm-jwt-for-user', + ); + } finally { + await stopServer(server); + } + }); + + test('end-to-end: realm-server-session → realm-create flow', async function (assert) { + let requests: CapturedRequest[] = []; + + let { server, origin } = await startTestServer((req, respond) => { + requests.push(req); + + if (req.url?.endsWith('_server-session')) { + respond(201, null, { + Authorization: 'Bearer e2e-realm-server-jwt', + }); + } else if (req.url?.endsWith('_create-realm')) { + respond(201, { + data: { type: 'realm', id: `${origin}/user/e2e-scratch/` }, + }); + } else { + respond(404, { error: 'not found' }); + } + }); + + try { + let registry = new ToolRegistry(); + let serverUrl = `${origin}/user/target/`; + + // Step 1: Obtain realm-server JWT + let sessionExecutor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: serverUrl, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + let sessionResult = await sessionExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-server-session', + toolArgs: { + 'realm-server-url': serverUrl, + 'openid-token': 'e2e-openid-token', + }, + }); + + assert.strictEqual(sessionResult.exitCode, 0); + let jwt = (sessionResult.output as { token: string }).token; + assert.strictEqual(jwt, 'Bearer e2e-realm-server-jwt'); + + // Step 2: Use JWT to create a realm + let createExecutor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: serverUrl, + testRealmUrl: `${origin}/user/target-tests/`, + authorization: jwt, + }); + + let createResult = await createExecutor.execute({ + type: 'invoke_tool', + tool: 'realm-create', + toolArgs: { + 'realm-server-url': serverUrl, + name: 'E2E Scratch', + endpoint: 'user/e2e-scratch', + }, + }); + + assert.strictEqual(createResult.exitCode, 0); + assert.strictEqual(requests.length, 2, 'two requests made'); + + // Verify the create request used the minted JWT + let createReq = requests[1]; + assert.strictEqual(createReq.method, 'POST'); + assert.strictEqual(createReq.url, '/user/target/_create-realm'); + assert.strictEqual( + createReq.headers.authorization, + 'Bearer e2e-realm-server-jwt', + 'create request uses the JWT from session', + ); + } finally { + await stopServer(server); + } + }); +}); + +// --------------------------------------------------------------------------- +// Integration tests: safety constraints prevent requests from reaching server +// --------------------------------------------------------------------------- + +module('factory-tool-executor integration > safety constraints', function () { + test('unregistered tool is rejected before any HTTP request', async function (assert) { + let requestCount = 0; + + let { server, origin } = await startTestServer((_req, respond) => { + requestCount++; + respond(200, {}); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'shell-exec-arbitrary', + toolArgs: { command: 'rm -rf /' }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + (err as Error).message.includes('Unregistered tool'), + 'throws for unregistered tool', + ); + } + + assert.strictEqual(requestCount, 0, 'server received zero requests'); + } finally { + await stopServer(server); + } + }); + + test('source realm targeting is rejected before any HTTP request', async function (assert) { + let requestCount = 0; + + let { server, origin } = await startTestServer((_req, respond) => { + requestCount++; + respond(200, {}); + }); + + try { + let registry = new ToolRegistry(); + let sourceUrl = `${origin}/user/source/`; + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + sourceRealmUrl: sourceUrl, + }); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'search-realm', + toolArgs: { realm: sourceUrl }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + (err as Error).message.includes('source realm'), + 'throws for source realm targeting', + ); + } + + assert.strictEqual(requestCount, 0, 'server received zero requests'); + } finally { + await stopServer(server); + } + }); + + test('unknown realm targeting is rejected before any HTTP request', async function (assert) { + let requestCount = 0; + + let { server, origin } = await startTestServer((_req, respond) => { + requestCount++; + respond(200, {}); + }); + + try { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: '/fake', + targetRealmUrl: `${origin}/user/target/`, + testRealmUrl: `${origin}/user/target-tests/`, + }); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': 'https://evil.example.test/hacker/realm/', + path: 'secrets.json', + }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + (err as Error).message.includes('not in the allowed list'), + 'throws for unknown realm', + ); + } + + assert.strictEqual(requestCount, 0, 'server received zero requests'); + } finally { + await stopServer(server); + } + }); +}); diff --git a/packages/software-factory/tests/index.ts b/packages/software-factory/tests/index.ts index 4201f58613a..298f2ac462a 100644 --- a/packages/software-factory/tests/index.ts +++ b/packages/software-factory/tests/index.ts @@ -5,5 +5,6 @@ import './factory-entrypoint.test'; import './factory-entrypoint.integration.test'; import './factory-target-realm.test'; import './factory-tool-executor.test'; +import './factory-tool-executor.integration.test'; import './factory-tool-registry.test'; import './realm-auth.test'; From e4eca7a826a8db06c01872059ccb3fe269bb57ee Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 18:31:30 -0400 Subject: [PATCH 05/13] Add realm-auth tool, realm-create defaults, and live integration tests - Add realm-auth tool: POST _realm-auth to get per-realm JWTs - Remove realm-mtimes and realm-reindex (agent won't use directly) - realm-create: default iconURL from name, random backgroundURL, update Matrix account data (app.boxel.realms) after create - realm-read: document card+source vs card+json accept header behavior - Add live integration test suite (pnpm test:live) for real server contract - Update unit and mock-server integration tests for all changes Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/software-factory/package.json | 1 + .../scripts/factory-tools-smoke.ts | 2 +- .../scripts/lib/factory-tool-executor.ts | 222 +++++++++++- .../scripts/lib/factory-tool-registry.ts | 41 ++- .../factory-tool-executor.integration.test.ts | 74 +--- .../tests/factory-tool-executor.live.test.ts | 324 ++++++++++++++++++ .../tests/factory-tool-executor.test.ts | 284 +++++++++++++-- .../tests/factory-tool-registry.test.ts | 3 +- packages/software-factory/tests/live-index.ts | 1 + 9 files changed, 827 insertions(+), 125 deletions(-) create mode 100644 packages/software-factory/tests/factory-tool-executor.live.test.ts create mode 100644 packages/software-factory/tests/live-index.ts diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index c64335dd371..9920f1ded2a 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -27,6 +27,7 @@ "test:node": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/test.ts --node-only", "test:playwright": "playwright test", "test:playwright:headed": "playwright test --headed", + "test:live": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/live-index.ts", "test:realm": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/run-realm-tests.ts" }, "devDependencies": { diff --git a/packages/software-factory/scripts/factory-tools-smoke.ts b/packages/software-factory/scripts/factory-tools-smoke.ts index ac89a0daae5..959f83714e4 100644 --- a/packages/software-factory/scripts/factory-tools-smoke.ts +++ b/packages/software-factory/scripts/factory-tools-smoke.ts @@ -80,7 +80,7 @@ async function main(): Promise { check('has script tools', byCategory['script']?.length === 4); check('has boxel-cli tools', byCategory['boxel-cli']?.length === 6); - check('has realm-api tools', byCategory['realm-api']?.length === 9); + check('has realm-api tools', byCategory['realm-api']?.length === 8); check( 'all names unique', new Set(manifests.map((m) => m.name)).size === manifests.length, diff --git a/packages/software-factory/scripts/lib/factory-tool-executor.ts b/packages/software-factory/scripts/lib/factory-tool-executor.ts index c99866ab58a..6487573771a 100644 --- a/packages/software-factory/scripts/lib/factory-tool-executor.ts +++ b/packages/software-factory/scripts/lib/factory-tool-executor.ts @@ -56,6 +56,12 @@ export interface ToolExecutorConfig { timeoutMs?: number; /** Optional log function for auditability. */ log?: (entry: ToolExecutionLogEntry) => void; + /** Matrix homeserver URL (for post-create account data update). */ + matrixUrl?: string; + /** Matrix access token (from Matrix login). */ + matrixAccessToken?: string; + /** Matrix user ID (e.g. @factory:localhost). */ + matrixUserId?: string; } export interface ToolExecutionLogEntry { @@ -388,6 +394,15 @@ export class ToolExecutor { responseBody = { token: authorizationHeader }; } + // After successful realm-create, update Matrix account data to include + // the new realm in the user's realm list (matching host app behavior). + if (toolName === 'realm-create' && response.ok) { + let createdRealmUrl = extractCreatedRealmUrl(responseBody); + if (createdRealmUrl) { + await this.appendRealmToMatrixAccountData(fetchImpl, createdRealmUrl); + } + } + return { tool: toolName, exitCode: response.ok ? 0 : 1, @@ -493,6 +508,63 @@ export class ToolExecutor { private logExecution(entry: ToolExecutionLogEntry): void { this.config.log?.(entry); } + + // ------------------------------------------------------------------------- + // Matrix account data + // ------------------------------------------------------------------------- + + /** + * After realm creation, append the new realm URL to the user's Matrix + * account data so the realm appears in their realm list. Matches the host + * app's `appendRealmToAccountData` behavior (matrix-service.ts:639-653). + * + * Silently skips when Matrix config is not provided. + */ + private async appendRealmToMatrixAccountData( + fetchImpl: typeof globalThis.fetch, + newRealmUrl: string, + ): Promise { + let { matrixUrl, matrixAccessToken, matrixUserId } = this.config; + if (!matrixUrl || !matrixAccessToken || !matrixUserId) { + return; + } + + let baseUrl = ensureTrailingSlash(matrixUrl); + let encodedUserId = encodeURIComponent(matrixUserId); + let accountDataUrl = `${baseUrl}_matrix/client/v3/user/${encodedUserId}/account_data/app.boxel.realms`; + let authHeaders = { + Authorization: `Bearer ${matrixAccessToken}`, + 'Content-Type': 'application/json', + }; + + // GET current realm list + let existingRealms: string[] = []; + try { + let getResponse = await fetchImpl(accountDataUrl, { + method: 'GET', + headers: authHeaders, + }); + if (getResponse.ok) { + let data = (await getResponse.json()) as { realms?: string[] }; + existingRealms = data.realms ?? []; + } + // 404 means no account data yet — start with empty list + } catch { + // Network error reading account data — proceed with empty list + } + + // Append and PUT + let updatedRealms = [...existingRealms, newRealmUrl]; + try { + await fetchImpl(accountDataUrl, { + method: 'PUT', + headers: authHeaders, + body: JSON.stringify({ realms: updatedRealms }), + }); + } catch { + // Best-effort — don't fail the realm-create if account data update fails + } + } } // --------------------------------------------------------------------------- @@ -715,19 +787,18 @@ function buildRealmApiRequest( }; } - case 'realm-mtimes': { - let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); - return { - url: `${realmUrl}_mtimes`, - method: 'GET', - headers, - }; - } - case 'realm-create': { let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); let name = String(toolArgs['name']); let endpoint = String(toolArgs['endpoint']); + let iconURL = + typeof toolArgs['iconURL'] === 'string' + ? toolArgs['iconURL'] + : iconURLForName(name); + let backgroundURL = + typeof toolArgs['backgroundURL'] === 'string' + ? toolArgs['backgroundURL'] + : getRandomBackgroundURL(); return { url: `${serverUrl}_create-realm`, method: 'POST', @@ -739,7 +810,12 @@ function buildRealmApiRequest( body: JSON.stringify({ data: { type: 'realm', - attributes: { name, endpoint }, + attributes: { + name, + endpoint, + ...(iconURL ? { iconURL } : {}), + ...(backgroundURL ? { backgroundURL } : {}), + }, }, }), }; @@ -756,12 +832,12 @@ function buildRealmApiRequest( }; } - case 'realm-reindex': { - let realmUrl = ensureTrailingSlash(String(toolArgs['realm-url'])); + case 'realm-auth': { + let serverUrl = ensureTrailingSlash(String(toolArgs['realm-server-url'])); return { - url: `${realmUrl}_reindex`, + url: `${serverUrl}_realm-auth`, method: 'POST', - headers, + headers: { ...headers, 'Content-Type': 'application/json' }, }; } @@ -781,3 +857,121 @@ function ensureTrailingSlash(url: string): string { function looksLikeUrl(value: string): boolean { return value.startsWith('http://') || value.startsWith('https://'); } + +function extractCreatedRealmUrl(responseBody: unknown): string | undefined { + if ( + typeof responseBody === 'object' && + responseBody !== null && + 'data' in responseBody + ) { + let data = (responseBody as { data: unknown }).data; + if (typeof data === 'object' && data !== null && 'id' in data) { + let id = (data as { id: unknown }).id; + if (typeof id === 'string') { + return id; + } + } + } + return undefined; +} + +// --------------------------------------------------------------------------- +// Icon and background URL defaults +// TODO: Move iconURLForName / getRandomBackgroundURL to @cardstack/runtime-common +// so host (packages/host/app/lib/utils.ts) and software-factory share one copy. +// --------------------------------------------------------------------------- + +const ICON_URLS: Record = { + a: 'https://boxel-images.boxel.ai/icons/Letter-a.png', + b: 'https://boxel-images.boxel.ai/icons/Letter-b.png', + c: 'https://boxel-images.boxel.ai/icons/Letter-c.png', + d: 'https://boxel-images.boxel.ai/icons/Letter-d.png', + e: 'https://boxel-images.boxel.ai/icons/Letter-e.png', + f: 'https://boxel-images.boxel.ai/icons/Letter-f.png', + g: 'https://boxel-images.boxel.ai/icons/Letter-g.png', + h: 'https://boxel-images.boxel.ai/icons/Letter-h.png', + i: 'https://boxel-images.boxel.ai/icons/Letter-i.png', + j: 'https://boxel-images.boxel.ai/icons/Letter-j.png', + k: 'https://boxel-images.boxel.ai/icons/Letter-k.png', + l: 'https://boxel-images.boxel.ai/icons/Letter-l.png', + m: 'https://boxel-images.boxel.ai/icons/Letter-m.png', + n: 'https://boxel-images.boxel.ai/icons/Letter-n.png', + o: 'https://boxel-images.boxel.ai/icons/Letter-o.png', + p: 'https://boxel-images.boxel.ai/icons/Letter-p.png', + q: 'https://boxel-images.boxel.ai/icons/Letter-q.png', + r: 'https://boxel-images.boxel.ai/icons/Letter-r.png', + s: 'https://boxel-images.boxel.ai/icons/Letter-s.png', + t: 'https://boxel-images.boxel.ai/icons/Letter-t.png', + u: 'https://boxel-images.boxel.ai/icons/Letter-u.png', + v: 'https://boxel-images.boxel.ai/icons/Letter-v.png', + w: 'https://boxel-images.boxel.ai/icons/Letter-w.png', + x: 'https://boxel-images.boxel.ai/icons/Letter-x.png', + y: 'https://boxel-images.boxel.ai/icons/Letter-y.png', + z: 'https://boxel-images.boxel.ai/icons/letter-z.png', +}; + +const BACKGROUND_URLS: readonly string[] = [ + 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', +]; + +export function iconURLForName(name: string): string | undefined { + if (!name) { + return undefined; + } + let cleansed = name + .toLowerCase() + .replace(/[^a-z0-9]/g, '') + .replace(/^[0-9]+/, ''); + return ICON_URLS[cleansed.charAt(0)]; +} + +export function getRandomBackgroundURL(): string { + let index = Math.floor(Math.random() * BACKGROUND_URLS.length); + return BACKGROUND_URLS[index]; +} diff --git a/packages/software-factory/scripts/lib/factory-tool-registry.ts b/packages/software-factory/scripts/lib/factory-tool-registry.ts index 3efd93622bb..a2bd31b9d0c 100644 --- a/packages/software-factory/scripts/lib/factory-tool-registry.ts +++ b/packages/software-factory/scripts/lib/factory-tool-registry.ts @@ -345,7 +345,9 @@ const REALM_API_TOOLS: ToolManifest[] = [ name: 'accept', type: 'string', required: false, - description: 'Accept header (default: application/vnd.card+source)', + description: + 'Accept header. Default: application/vnd.card+source (raw source — path MUST include file extension, e.g. CardDef/my-card.gts). ' + + 'Use application/vnd.card+json for computed card instances with resolved fields (path must NOT include extension, e.g. Card/instance).', }, ], }, @@ -443,20 +445,6 @@ const REALM_API_TOOLS: ToolManifest[] = [ }, ], }, - { - name: 'realm-mtimes', - description: 'Get file modification times for a realm.', - category: 'realm-api', - outputFormat: 'json', - args: [ - { - name: 'realm-url', - type: 'string', - required: true, - description: 'Realm base URL', - }, - ], - }, { name: 'realm-create', description: 'Create a new realm on the realm server.', @@ -482,6 +470,20 @@ const REALM_API_TOOLS: ToolManifest[] = [ description: 'URL path segment for the new realm (e.g. "user/my-realm")', }, + { + name: 'iconURL', + type: 'string', + required: false, + description: + 'Icon URL for the realm. Defaults to a letter-based icon derived from the realm name.', + }, + { + name: 'backgroundURL', + type: 'string', + required: false, + description: + 'Background image URL for the realm. Defaults to a random background image.', + }, ], }, { @@ -507,16 +509,17 @@ const REALM_API_TOOLS: ToolManifest[] = [ ], }, { - name: 'realm-reindex', - description: 'Trigger a full reindex of a realm.', + name: 'realm-auth', + description: + 'Get per-realm JWTs for all realms accessible to the authenticated user.', category: 'realm-api', outputFormat: 'json', args: [ { - name: 'realm-url', + name: 'realm-server-url', type: 'string', required: true, - description: 'Realm URL to reindex', + description: 'Realm server base URL', }, ], }, diff --git a/packages/software-factory/tests/factory-tool-executor.integration.test.ts b/packages/software-factory/tests/factory-tool-executor.integration.test.ts index d48679b7c93..121edab89e4 100644 --- a/packages/software-factory/tests/factory-tool-executor.integration.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.integration.test.ts @@ -303,36 +303,37 @@ module('factory-tool-executor integration > realm-api requests', function () { } }); - test('realm-mtimes sends correct GET to _mtimes', async function (assert) { + test('realm-auth sends correct POST to _realm-auth with Authorization header', async function (assert) { let captured: CapturedRequest | undefined; let { server, origin } = await startTestServer((req, respond) => { captured = req; - respond(200, { 'Card/foo.json': 1700000000 }); + respond(200, { ok: true }); }); try { let registry = new ToolRegistry(); - let realmUrl = `${origin}/user/target/`; let executor = new ToolExecutor(registry, { packageRoot: '/fake', - targetRealmUrl: realmUrl, + targetRealmUrl: `${origin}/user/target/`, testRealmUrl: `${origin}/user/target-tests/`, - authorization: 'Bearer realm-jwt-for-user', + authorization: 'Bearer realm-server-jwt-xyz', }); let result = await executor.execute({ type: 'invoke_tool', - tool: 'realm-mtimes', - toolArgs: { 'realm-url': realmUrl }, + tool: 'realm-auth', + toolArgs: { + 'realm-server-url': `${origin}/user/target/`, + }, }); assert.strictEqual(result.exitCode, 0); - assert.strictEqual(captured!.method, 'GET'); - assert.strictEqual(captured!.url, '/user/target/_mtimes'); + assert.strictEqual(captured!.method, 'POST'); + assert.strictEqual(captured!.url, '/user/target/_realm-auth'); assert.strictEqual( captured!.headers.authorization, - 'Bearer realm-jwt-for-user', + 'Bearer realm-server-jwt-xyz', ); } finally { await stopServer(server); @@ -382,15 +383,14 @@ module('factory-tool-executor integration > realm-api requests', function () { ); let body = JSON.parse(captured!.body); - assert.deepEqual(body, { - data: { - type: 'realm', - attributes: { - name: 'New Realm', - endpoint: 'user/new-realm', - }, - }, - }); + assert.strictEqual(body.data.type, 'realm'); + assert.strictEqual(body.data.attributes.name, 'New Realm'); + assert.strictEqual(body.data.attributes.endpoint, 'user/new-realm'); + assert.ok(body.data.attributes.iconURL, 'body includes iconURL'); + assert.ok( + body.data.attributes.backgroundURL, + 'body includes backgroundURL', + ); } finally { await stopServer(server); } @@ -445,42 +445,6 @@ module('factory-tool-executor integration > realm-api requests', function () { } }); - test('realm-reindex sends correct POST to _reindex', async function (assert) { - let captured: CapturedRequest | undefined; - - let { server, origin } = await startTestServer((req, respond) => { - captured = req; - respond(200, { status: 'reindexing' }); - }); - - try { - let registry = new ToolRegistry(); - let realmUrl = `${origin}/user/target/`; - let executor = new ToolExecutor(registry, { - packageRoot: '/fake', - targetRealmUrl: realmUrl, - testRealmUrl: `${origin}/user/target-tests/`, - authorization: 'Bearer realm-jwt-for-user', - }); - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-reindex', - toolArgs: { 'realm-url': realmUrl }, - }); - - assert.strictEqual(result.exitCode, 0); - assert.strictEqual(captured!.method, 'POST'); - assert.strictEqual(captured!.url, '/user/target/_reindex'); - assert.strictEqual( - captured!.headers.authorization, - 'Bearer realm-jwt-for-user', - ); - } finally { - await stopServer(server); - } - }); - test('end-to-end: realm-server-session → realm-create flow', async function (assert) { let requests: CapturedRequest[] = []; diff --git a/packages/software-factory/tests/factory-tool-executor.live.test.ts b/packages/software-factory/tests/factory-tool-executor.live.test.ts new file mode 100644 index 00000000000..109e983234e --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.live.test.ts @@ -0,0 +1,324 @@ +/** + * Live integration tests for the ToolExecutor against a running realm server. + * + * These tests hit the real realm server APIs to verify the tool executor + * produces requests the server accepts and responses match expected shapes. + * + * Prerequisites (run in separate terminals): + * 1. pnpm serve:support # starts Matrix, Postgres, prerender + * 2. pnpm cache:prepare # creates template database + * 3. pnpm serve:realm # starts realm server on port 4205 + * + * Or use the dev stack from the repo root: + * mise run dev # starts everything + * + * Required env vars: + * MATRIX_URL — Matrix homeserver URL (e.g. http://localhost:8008/) + * MATRIX_USERNAME — Matrix username (e.g. the software-factory owner) + * MATRIX_PASSWORD — Matrix password + * REALM_SERVER_URL — Realm server base URL (e.g. http://localhost:4205/) + * + * Run: + * MATRIX_URL=http://localhost:8008/ MATRIX_USERNAME=... MATRIX_PASSWORD=... \ + * REALM_SERVER_URL=http://localhost:4205/ pnpm test:live + * + * Tests skip gracefully when env vars are not set. + */ + +import { module, test } from 'qunit'; + +import { + getOpenIdToken, + matrixLogin, + type MatrixAuth, +} from '../scripts/lib/boxel'; +import { + ToolExecutor, + ToolNotFoundError, + type ToolExecutorConfig, +} from '../scripts/lib/factory-tool-executor'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +// --------------------------------------------------------------------------- +// Env var check +// --------------------------------------------------------------------------- + +let matrixUrl = process.env.MATRIX_URL?.trim(); +let matrixUsername = process.env.MATRIX_USERNAME?.trim(); +let matrixPassword = process.env.MATRIX_PASSWORD?.trim(); +let realmServerUrl = process.env.REALM_SERVER_URL?.trim(); + +let hasLiveConfig = !!( + matrixUrl && + matrixUsername && + matrixPassword && + realmServerUrl +); + +// --------------------------------------------------------------------------- +// Helpers (must be at module scope for eslint no-inner-declarations) +// --------------------------------------------------------------------------- + +function ensureTrailingSlash(url: string): string { + return url.endsWith('/') ? url : `${url}/`; +} + +// Shared auth state — populated in hooks.before +let matrixAuth: MatrixAuth; +let openIdTokenJson: string; +let serverJwt: string; +let realmJwts: Record; +let firstRealmUrl: string; + +function makeExecutorConfig( + overrides?: Partial, +): ToolExecutorConfig { + return { + packageRoot: process.cwd(), + targetRealmUrl: firstRealmUrl ?? ensureTrailingSlash(realmServerUrl ?? ''), + testRealmUrl: ensureTrailingSlash(realmServerUrl ?? ''), + allowedRealmPrefixes: [ensureTrailingSlash(realmServerUrl ?? '')], + ...overrides, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +if (!hasLiveConfig) { + module('factory-tool-executor live (SKIPPED)', function () { + test('live tests skipped — set MATRIX_URL, MATRIX_USERNAME, MATRIX_PASSWORD, REALM_SERVER_URL to enable', function (assert) { + assert.true( + true, + 'Skipped: env vars not set. Run with pnpm test:live after setting env.', + ); + }); + }); +} else { + module('factory-tool-executor live', function (hooks) { + hooks.before(async function () { + matrixAuth = await matrixLogin({ + profileId: null, + username: matrixUsername!, + matrixUrl: ensureTrailingSlash(matrixUrl!), + realmServerUrl: ensureTrailingSlash(realmServerUrl!), + password: matrixPassword!, + }); + + let openIdToken = await getOpenIdToken(matrixAuth); + openIdTokenJson = JSON.stringify(openIdToken); + }); + + test('realm-server-session returns a server JWT', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeExecutorConfig()); + + let tokenPayload = JSON.parse(openIdTokenJson) as { + access_token: string; + }; + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-server-session', + toolArgs: { + 'realm-server-url': realmServerUrl!, + 'openid-token': tokenPayload.access_token, + }, + }); + + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + let output = result.output as { token: string }; + assert.strictEqual(typeof output.token, 'string', 'token is a string'); + assert.true( + output.token.startsWith('Bearer '), + 'token starts with Bearer', + ); + + serverJwt = output.token; + }); + + test('realm-auth returns per-realm JWT map', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ authorization: serverJwt }), + ); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-auth', + toolArgs: { 'realm-server-url': realmServerUrl! }, + }); + + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + + let output = result.output as Record; + let realmUrls = Object.keys(output); + assert.true(realmUrls.length > 0, 'at least one realm in JWT map'); + + for (let [url, jwt] of Object.entries(output)) { + assert.true(url.startsWith('http'), `key "${url}" looks like a URL`); + assert.strictEqual(typeof jwt, 'string', `JWT for ${url} is a string`); + } + + realmJwts = output; + firstRealmUrl = realmUrls[0]; + }); + + test('realm-read fetches .realm.json from an accessible realm', async function (assert) { + let realmJwt = realmJwts[firstRealmUrl]; + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ authorization: realmJwt }), + ); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': firstRealmUrl, + path: '.realm.json', + }, + }); + + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + assert.strictEqual(typeof result.output, 'object', 'output is an object'); + }); + + test('realm-search returns results', async function (assert) { + let realmJwt = realmJwts[firstRealmUrl]; + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ authorization: realmJwt }), + ); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': firstRealmUrl, + query: JSON.stringify({ + filter: {}, + page: { size: 1 }, + }), + }, + }); + + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + let output = result.output as { data?: unknown[] }; + assert.true(Array.isArray(output.data), 'output has data array'); + }); + + test('realm-create creates a realm with icon and background', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ + authorization: serverJwt, + matrixUrl: ensureTrailingSlash(matrixUrl!), + matrixAccessToken: matrixAuth.accessToken, + matrixUserId: matrixAuth.userId, + }), + ); + + let timestamp = Date.now(); + let endpoint = `live-test-${timestamp}`; + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-create', + toolArgs: { + 'realm-server-url': realmServerUrl!, + name: `Live Test ${timestamp}`, + endpoint, + }, + }); + + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + + let output = result.output as { + data?: { + type?: string; + id?: string; + attributes?: Record; + }; + }; + assert.strictEqual(output.data?.type, 'realm', 'response type is realm'); + assert.strictEqual( + typeof output.data?.id, + 'string', + 'response has realm id', + ); + + // Verify Matrix account data was updated + let encodedUserId = encodeURIComponent(matrixAuth.userId); + let accountDataUrl = + `${ensureTrailingSlash(matrixUrl!)}` + + `_matrix/client/v3/user/${encodedUserId}/account_data/app.boxel.realms`; + + let accountDataResponse = await fetch(accountDataUrl, { + headers: { Authorization: `Bearer ${matrixAuth.accessToken}` }, + }); + + if (accountDataResponse.ok) { + let accountData = (await accountDataResponse.json()) as { + realms?: string[]; + }; + let createdRealmUrl = output.data?.id; + let realmFound = + accountData.realms?.some((r) => r === createdRealmUrl) ?? false; + assert.true( + realmFound, + `Matrix account data includes newly created realm ${createdRealmUrl}`, + ); + } else { + assert.true( + true, + `Could not verify Matrix account data (status ${accountDataResponse.status})`, + ); + } + }); + + test('unregistered tool is rejected without making HTTP calls', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeExecutorConfig()); + + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'shell-exec-arbitrary', + toolArgs: { command: 'rm -rf /' }, + }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true( + err instanceof ToolNotFoundError, + 'throws ToolNotFoundError', + ); + } + }); + }); +} diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts index 6dcfffdf7aa..7da8fc5b7b5 100644 --- a/packages/software-factory/tests/factory-tool-executor.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -6,6 +6,7 @@ import { ToolNotFoundError, ToolSafetyError, ToolTimeoutError, + iconURLForName, type ToolExecutionLogEntry, type ToolExecutorConfig, } from '../scripts/lib/factory-tool-executor'; @@ -449,7 +450,7 @@ module('factory-tool-executor > realm-api execution', function () { ); }); - test('realm-mtimes makes GET to _mtimes', async function (assert) { + test('realm-auth makes POST to _realm-auth', async function (assert) { let capturedUrl: string | undefined; let capturedMethod: string | undefined; @@ -458,7 +459,7 @@ module('factory-tool-executor > realm-api execution', function () { fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { capturedUrl = String(input); capturedMethod = init?.method; - return new Response(JSON.stringify({ 'foo.json': 12345 }), { + return new Response(JSON.stringify({ ok: true }), { status: 200, headers: { 'Content-Type': 'application/json' }, }); @@ -467,13 +468,13 @@ module('factory-tool-executor > realm-api execution', function () { let executor = new ToolExecutor(registry, config); let result = await executor.execute( - makeInvokeToolAction('realm-mtimes', { - 'realm-url': 'https://realms.example.test/user/target/', + makeInvokeToolAction('realm-auth', { + 'realm-server-url': 'https://realms.example.test/user/target/', }), ); - assert.strictEqual(capturedMethod, 'GET'); - assert.true(capturedUrl!.endsWith('_mtimes')); + assert.strictEqual(capturedMethod, 'POST'); + assert.true(capturedUrl!.endsWith('_realm-auth')); assert.strictEqual(result.exitCode, 0); }); @@ -486,10 +487,18 @@ module('factory-tool-executor > realm-api execution', function () { fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { capturedUrl = String(input); capturedBody = typeof init?.body === 'string' ? init.body : undefined; - return new Response(JSON.stringify({ ok: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch-123/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }) as typeof globalThis.fetch, }); let executor = new ToolExecutor(registry, config); @@ -507,28 +516,175 @@ module('factory-tool-executor > realm-api execution', function () { 'https://realms.example.test/_create-realm', ); let body = JSON.parse(capturedBody!); - assert.deepEqual(body, { - data: { - type: 'realm', - attributes: { - name: 'my-scratch-realm', - endpoint: 'user/scratch-123', - }, - }, - }); + assert.strictEqual(body.data.type, 'realm'); + assert.strictEqual(body.data.attributes.name, 'my-scratch-realm'); + assert.strictEqual(body.data.attributes.endpoint, 'user/scratch-123'); + assert.ok(body.data.attributes.iconURL, 'iconURL is present'); + assert.ok(body.data.attributes.backgroundURL, 'backgroundURL is present'); assert.strictEqual(result.exitCode, 0); }); - test('realm-reindex makes POST to _reindex', async function (assert) { - let capturedUrl: string | undefined; - let capturedMethod: string | undefined; + test('realm-create with explicit iconURL and backgroundURL', async function (assert) { + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'my-realm', + endpoint: 'user/scratch', + iconURL: 'https://example.test/icon.png', + backgroundURL: 'https://example.test/bg.jpg', + }), + ); + + let body = JSON.parse(capturedBody!); + assert.strictEqual( + body.data.attributes.iconURL, + 'https://example.test/icon.png', + ); + assert.strictEqual( + body.data.attributes.backgroundURL, + 'https://example.test/bg.jpg', + ); + }); + + test('realm-create applies default icon from name', async function (assert) { + let capturedBody: string | undefined; + + let registry = new ToolRegistry(); + let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'My Realm', + endpoint: 'user/scratch', + }), + ); + + let body = JSON.parse(capturedBody!); + assert.strictEqual( + body.data.attributes.iconURL, + iconURLForName('My Realm'), + 'iconURL defaults from name', + ); + }); + + test('realm-create applies default random background', async function (assert) { + let capturedBody: string | undefined; let registry = new ToolRegistry(); let config = makeConfig({ + fetch: (async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedBody = typeof init?.body === 'string' ? init.body : undefined; + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'My Realm', + endpoint: 'user/scratch', + }), + ); + + let body = JSON.parse(capturedBody!); + assert.true( + body.data.attributes.backgroundURL.startsWith( + 'https://boxel-images.boxel.ai/background-images/', + ), + 'backgroundURL defaults to a random background', + ); + }); + + test('realm-create updates Matrix account data when config present', async function (assert) { + let fetchCalls: { url: string; method: string }[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + matrixUrl: 'https://matrix.example.test', + matrixAccessToken: 'matrix-token-123', + matrixUserId: '@factory:example.test', fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { - capturedUrl = String(input); - capturedMethod = init?.method; - return new Response(JSON.stringify({ ok: true }), { + let url = String(input); + let method = init?.method ?? 'GET'; + fetchCalls.push({ url, method }); + + if (url.includes('_create-realm')) { + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + } + if (url.includes('account_data') && method === 'GET') { + return new Response( + JSON.stringify({ realms: ['https://existing.test/'] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + } + // PUT account_data + return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' }, }); @@ -536,15 +692,75 @@ module('factory-tool-executor > realm-api execution', function () { }); let executor = new ToolExecutor(registry, config); - let result = await executor.execute( - makeInvokeToolAction('realm-reindex', { - 'realm-url': 'https://realms.example.test/user/target/', + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'scratch', + endpoint: 'user/scratch', }), ); - assert.strictEqual(capturedMethod, 'POST'); - assert.true(capturedUrl!.endsWith('_reindex')); - assert.strictEqual(result.exitCode, 0); + assert.strictEqual(fetchCalls.length, 3, 'three fetch calls made'); + assert.true( + fetchCalls[0].url.includes('_create-realm'), + 'first call is _create-realm', + ); + assert.true( + fetchCalls[1].url.includes('account_data'), + 'second call is Matrix GET account_data', + ); + assert.strictEqual(fetchCalls[1].method, 'GET'); + assert.true( + fetchCalls[2].url.includes('account_data'), + 'third call is Matrix PUT account_data', + ); + assert.strictEqual(fetchCalls[2].method, 'PUT'); + }); + + test('realm-create skips Matrix update when config absent', async function (assert) { + let fetchCalls: { url: string; method: string }[] = []; + + let registry = new ToolRegistry(); + let config = makeConfig({ + // No matrixUrl, matrixAccessToken, or matrixUserId + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + let url = String(input); + let method = init?.method ?? 'GET'; + fetchCalls.push({ url, method }); + + return new Response( + JSON.stringify({ + data: { + type: 'realm', + id: 'https://realms.example.test/user/scratch/', + }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }) as typeof globalThis.fetch, + }); + let executor = new ToolExecutor(registry, config); + + await executor.execute( + makeInvokeToolAction('realm-create', { + 'realm-server-url': 'https://realms.example.test/', + name: 'scratch', + endpoint: 'user/scratch', + }), + ); + + assert.strictEqual( + fetchCalls.length, + 1, + 'only one fetch call (no Matrix update)', + ); + assert.true( + fetchCalls[0].url.includes('_create-realm'), + 'only call is _create-realm', + ); }); test('realm-server-session sends OpenID token and captures Authorization header', async function (assert) { @@ -743,7 +959,7 @@ module('factory-tool-executor > auth header propagation', function () { ); }); - test('realm-reindex sends realm JWT in Authorization header', async function (assert) { + test('realm-auth sends server JWT in Authorization header', async function (assert) { let { fetch, getCapturedHeaders } = createHeaderCapturingFetch(); let registry = new ToolRegistry(); let executor = new ToolExecutor( @@ -752,8 +968,8 @@ module('factory-tool-executor > auth header propagation', function () { ); await executor.execute( - makeInvokeToolAction('realm-reindex', { - 'realm-url': 'https://realms.example.test/user/target/', + makeInvokeToolAction('realm-auth', { + 'realm-server-url': 'https://realms.example.test/user/target/', }), ); diff --git a/packages/software-factory/tests/factory-tool-registry.test.ts b/packages/software-factory/tests/factory-tool-registry.test.ts index 6bcfb5fa7b7..d3fc3f6350f 100644 --- a/packages/software-factory/tests/factory-tool-registry.test.ts +++ b/packages/software-factory/tests/factory-tool-registry.test.ts @@ -286,10 +286,9 @@ module('factory-tool-registry > built-in manifests', function () { assert.true(registry.has('realm-delete')); assert.true(registry.has('realm-atomic')); assert.true(registry.has('realm-search')); - assert.true(registry.has('realm-mtimes')); assert.true(registry.has('realm-create')); assert.true(registry.has('realm-server-session')); - assert.true(registry.has('realm-reindex')); + assert.true(registry.has('realm-auth')); }); }); diff --git a/packages/software-factory/tests/live-index.ts b/packages/software-factory/tests/live-index.ts new file mode 100644 index 00000000000..a13f1d551c4 --- /dev/null +++ b/packages/software-factory/tests/live-index.ts @@ -0,0 +1 @@ +import './factory-tool-executor.live.test'; From 1a0cf6731788edd90c699a0024c9fdb778b9f234 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 18:35:20 -0400 Subject: [PATCH 06/13] Simplify live tests: use harness JWTs, run in pnpm test:node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live tests now mint JWTs directly using the harness secret seed (matching src/harness.ts pattern) instead of requiring Matrix login. They skip gracefully when the realm server isn't running and are part of the normal test:node suite — no separate pnpm script needed. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/software-factory/package.json | 1 - .../tests/factory-tool-executor.live.test.ts | 301 +++++++----------- packages/software-factory/tests/index.ts | 1 + packages/software-factory/tests/live-index.ts | 1 - 4 files changed, 108 insertions(+), 196 deletions(-) delete mode 100644 packages/software-factory/tests/live-index.ts diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index 9920f1ded2a..c64335dd371 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -27,7 +27,6 @@ "test:node": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/test.ts --node-only", "test:playwright": "playwright test", "test:playwright:headed": "playwright test --headed", - "test:live": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/live-index.ts", "test:realm": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/run-realm-tests.ts" }, "devDependencies": { diff --git a/packages/software-factory/tests/factory-tool-executor.live.test.ts b/packages/software-factory/tests/factory-tool-executor.live.test.ts index 109e983234e..e3f6389cdb1 100644 --- a/packages/software-factory/tests/factory-tool-executor.live.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.live.test.ts @@ -1,37 +1,24 @@ /** - * Live integration tests for the ToolExecutor against a running realm server. + * Live integration tests for the ToolExecutor against the software-factory + * harness realm server. * * These tests hit the real realm server APIs to verify the tool executor * produces requests the server accepts and responses match expected shapes. * - * Prerequisites (run in separate terminals): + * Prerequisites: * 1. pnpm serve:support # starts Matrix, Postgres, prerender * 2. pnpm cache:prepare # creates template database * 3. pnpm serve:realm # starts realm server on port 4205 * - * Or use the dev stack from the repo root: - * mise run dev # starts everything + * Auth uses the harness's known secret seed to mint JWTs directly, + * matching the pattern in src/harness.ts — no Matrix login needed. * - * Required env vars: - * MATRIX_URL — Matrix homeserver URL (e.g. http://localhost:8008/) - * MATRIX_USERNAME — Matrix username (e.g. the software-factory owner) - * MATRIX_PASSWORD — Matrix password - * REALM_SERVER_URL — Realm server base URL (e.g. http://localhost:4205/) - * - * Run: - * MATRIX_URL=http://localhost:8008/ MATRIX_USERNAME=... MATRIX_PASSWORD=... \ - * REALM_SERVER_URL=http://localhost:4205/ pnpm test:live - * - * Tests skip gracefully when env vars are not set. + * Tests skip gracefully when the realm server is not running. */ +import jwt from 'jsonwebtoken'; import { module, test } from 'qunit'; -import { - getOpenIdToken, - matrixLogin, - type MatrixAuth, -} from '../scripts/lib/boxel'; import { ToolExecutor, ToolNotFoundError, @@ -40,142 +27,98 @@ import { import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; // --------------------------------------------------------------------------- -// Env var check +// Harness constants (match src/harness.ts) // --------------------------------------------------------------------------- -let matrixUrl = process.env.MATRIX_URL?.trim(); -let matrixUsername = process.env.MATRIX_USERNAME?.trim(); -let matrixPassword = process.env.MATRIX_PASSWORD?.trim(); -let realmServerUrl = process.env.REALM_SERVER_URL?.trim(); - -let hasLiveConfig = !!( - matrixUrl && - matrixUsername && - matrixPassword && - realmServerUrl +const REALM_SERVER_PORT = Number( + process.env.SOFTWARE_FACTORY_REALM_PORT ?? 4205, ); +const REALM_SERVER_URL = `http://localhost:${REALM_SERVER_PORT}/`; +const TEST_REALM_URL = `${REALM_SERVER_URL}test/`; +const REALM_SECRET_SEED = "shhh! it's a secret"; +const REALM_SERVER_SECRET_SEED = "mum's the word"; +const DEFAULT_REALM_OWNER = '@software-factory-owner:localhost'; // --------------------------------------------------------------------------- -// Helpers (must be at module scope for eslint no-inner-declarations) +// Helpers // --------------------------------------------------------------------------- -function ensureTrailingSlash(url: string): string { - return url.endsWith('/') ? url : `${url}/`; +function buildRealmToken( + realmURL: string, + user = DEFAULT_REALM_OWNER, + permissions = ['read', 'write', 'realm-owner'], +): string { + return jwt.sign( + { + user, + realm: realmURL, + permissions, + sessionRoom: `software-factory-session-room-for-${user}`, + realmServerURL: REALM_SERVER_URL, + }, + REALM_SECRET_SEED, + { expiresIn: '7d' }, + ); } -// Shared auth state — populated in hooks.before -let matrixAuth: MatrixAuth; -let openIdTokenJson: string; -let serverJwt: string; -let realmJwts: Record; -let firstRealmUrl: string; +function buildRealmServerToken(user = DEFAULT_REALM_OWNER): string { + return jwt.sign( + { + user, + sessionRoom: `software-factory-session-room-for-${user}`, + }, + REALM_SERVER_SECRET_SEED, + { expiresIn: '7d' }, + ); +} function makeExecutorConfig( overrides?: Partial, ): ToolExecutorConfig { return { packageRoot: process.cwd(), - targetRealmUrl: firstRealmUrl ?? ensureTrailingSlash(realmServerUrl ?? ''), - testRealmUrl: ensureTrailingSlash(realmServerUrl ?? ''), - allowedRealmPrefixes: [ensureTrailingSlash(realmServerUrl ?? '')], + targetRealmUrl: TEST_REALM_URL, + testRealmUrl: TEST_REALM_URL, + allowedRealmPrefixes: [REALM_SERVER_URL], ...overrides, }; } +async function isRealmServerRunning(): Promise { + try { + let response = await fetch(REALM_SERVER_URL, { + method: 'HEAD', + signal: AbortSignal.timeout(2000), + }); + return response.ok || response.status === 403 || response.status === 404; + } catch { + return false; + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- -if (!hasLiveConfig) { - module('factory-tool-executor live (SKIPPED)', function () { - test('live tests skipped — set MATRIX_URL, MATRIX_USERNAME, MATRIX_PASSWORD, REALM_SERVER_URL to enable', function (assert) { - assert.true( - true, - 'Skipped: env vars not set. Run with pnpm test:live after setting env.', - ); - }); - }); -} else { - module('factory-tool-executor live', function (hooks) { - hooks.before(async function () { - matrixAuth = await matrixLogin({ - profileId: null, - username: matrixUsername!, - matrixUrl: ensureTrailingSlash(matrixUrl!), - realmServerUrl: ensureTrailingSlash(realmServerUrl!), - password: matrixPassword!, - }); - - let openIdToken = await getOpenIdToken(matrixAuth); - openIdTokenJson = JSON.stringify(openIdToken); - }); - - test('realm-server-session returns a server JWT', async function (assert) { - let registry = new ToolRegistry(); - let executor = new ToolExecutor(registry, makeExecutorConfig()); - - let tokenPayload = JSON.parse(openIdTokenJson) as { - access_token: string; - }; +module('factory-tool-executor live', function (hooks) { + let serverRunning = false; - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-server-session', - toolArgs: { - 'realm-server-url': realmServerUrl!, - 'openid-token': tokenPayload.access_token, - }, - }); - - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, + hooks.before(async function () { + serverRunning = await isRealmServerRunning(); + if (!serverRunning) { + console.log( + '\n [SKIP] Realm server not running at ' + + REALM_SERVER_URL + + ' — run: pnpm serve:support && pnpm cache:prepare && pnpm serve:realm\n', ); - let output = result.output as { token: string }; - assert.strictEqual(typeof output.token, 'string', 'token is a string'); - assert.true( - output.token.startsWith('Bearer '), - 'token starts with Bearer', - ); - - serverJwt = output.token; - }); - - test('realm-auth returns per-realm JWT map', async function (assert) { - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: serverJwt }), - ); - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-auth', - toolArgs: { 'realm-server-url': realmServerUrl! }, - }); - - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); - - let output = result.output as Record; - let realmUrls = Object.keys(output); - assert.true(realmUrls.length > 0, 'at least one realm in JWT map'); - - for (let [url, jwt] of Object.entries(output)) { - assert.true(url.startsWith('http'), `key "${url}" looks like a URL`); - assert.strictEqual(typeof jwt, 'string', `JWT for ${url} is a string`); - } - - realmJwts = output; - firstRealmUrl = realmUrls[0]; - }); + } + }); - test('realm-read fetches .realm.json from an accessible realm', async function (assert) { - let realmJwt = realmJwts[firstRealmUrl]; + test('realm-read fetches .realm.json from the test realm', async function (assert) { + if (!serverRunning) { + assert.expect(0); + } else { + let realmJwt = buildRealmToken(TEST_REALM_URL); let registry = new ToolRegistry(); let executor = new ToolExecutor( registry, @@ -186,7 +129,7 @@ if (!hasLiveConfig) { type: 'invoke_tool', tool: 'realm-read', toolArgs: { - 'realm-url': firstRealmUrl, + 'realm-url': TEST_REALM_URL, path: '.realm.json', }, }); @@ -197,10 +140,14 @@ if (!hasLiveConfig) { `exitCode 0, got: ${JSON.stringify(result.output)}`, ); assert.strictEqual(typeof result.output, 'object', 'output is an object'); - }); + } + }); - test('realm-search returns results', async function (assert) { - let realmJwt = realmJwts[firstRealmUrl]; + test('realm-search returns results from the test realm', async function (assert) { + if (!serverRunning) { + assert.expect(0); + } else { + let realmJwt = buildRealmToken(TEST_REALM_URL); let registry = new ToolRegistry(); let executor = new ToolExecutor( registry, @@ -211,11 +158,8 @@ if (!hasLiveConfig) { type: 'invoke_tool', tool: 'realm-search', toolArgs: { - 'realm-url': firstRealmUrl, - query: JSON.stringify({ - filter: {}, - page: { size: 1 }, - }), + 'realm-url': TEST_REALM_URL, + query: JSON.stringify({ filter: {}, page: { size: 1 } }), }, }); @@ -226,18 +170,18 @@ if (!hasLiveConfig) { ); let output = result.output as { data?: unknown[] }; assert.true(Array.isArray(output.data), 'output has data array'); - }); + } + }); - test('realm-create creates a realm with icon and background', async function (assert) { + test('realm-create creates a scratch realm with icon and background', async function (assert) { + if (!serverRunning) { + assert.expect(0); + } else { + let serverJwt = buildRealmServerToken(); let registry = new ToolRegistry(); let executor = new ToolExecutor( registry, - makeExecutorConfig({ - authorization: serverJwt, - matrixUrl: ensureTrailingSlash(matrixUrl!), - matrixAccessToken: matrixAuth.accessToken, - matrixUserId: matrixAuth.userId, - }), + makeExecutorConfig({ authorization: serverJwt }), ); let timestamp = Date.now(); @@ -247,7 +191,7 @@ if (!hasLiveConfig) { type: 'invoke_tool', tool: 'realm-create', toolArgs: { - 'realm-server-url': realmServerUrl!, + 'realm-server-url': REALM_SERVER_URL, name: `Live Test ${timestamp}`, endpoint, }, @@ -272,53 +216,22 @@ if (!hasLiveConfig) { 'string', 'response has realm id', ); + } + }); - // Verify Matrix account data was updated - let encodedUserId = encodeURIComponent(matrixAuth.userId); - let accountDataUrl = - `${ensureTrailingSlash(matrixUrl!)}` + - `_matrix/client/v3/user/${encodedUserId}/account_data/app.boxel.realms`; + test('unregistered tool is rejected', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeExecutorConfig()); - let accountDataResponse = await fetch(accountDataUrl, { - headers: { Authorization: `Bearer ${matrixAuth.accessToken}` }, + try { + await executor.execute({ + type: 'invoke_tool', + tool: 'shell-exec-arbitrary', + toolArgs: { command: 'rm -rf /' }, }); - - if (accountDataResponse.ok) { - let accountData = (await accountDataResponse.json()) as { - realms?: string[]; - }; - let createdRealmUrl = output.data?.id; - let realmFound = - accountData.realms?.some((r) => r === createdRealmUrl) ?? false; - assert.true( - realmFound, - `Matrix account data includes newly created realm ${createdRealmUrl}`, - ); - } else { - assert.true( - true, - `Could not verify Matrix account data (status ${accountDataResponse.status})`, - ); - } - }); - - test('unregistered tool is rejected without making HTTP calls', async function (assert) { - let registry = new ToolRegistry(); - let executor = new ToolExecutor(registry, makeExecutorConfig()); - - try { - await executor.execute({ - type: 'invoke_tool', - tool: 'shell-exec-arbitrary', - toolArgs: { command: 'rm -rf /' }, - }); - assert.ok(false, 'should have thrown'); - } catch (err) { - assert.true( - err instanceof ToolNotFoundError, - 'throws ToolNotFoundError', - ); - } - }); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolNotFoundError, 'throws ToolNotFoundError'); + } }); -} +}); diff --git a/packages/software-factory/tests/index.ts b/packages/software-factory/tests/index.ts index 298f2ac462a..42cb123a847 100644 --- a/packages/software-factory/tests/index.ts +++ b/packages/software-factory/tests/index.ts @@ -6,5 +6,6 @@ import './factory-entrypoint.integration.test'; import './factory-target-realm.test'; import './factory-tool-executor.test'; import './factory-tool-executor.integration.test'; +import './factory-tool-executor.live.test'; import './factory-tool-registry.test'; import './realm-auth.test'; diff --git a/packages/software-factory/tests/live-index.ts b/packages/software-factory/tests/live-index.ts deleted file mode 100644 index a13f1d551c4..00000000000 --- a/packages/software-factory/tests/live-index.ts +++ /dev/null @@ -1 +0,0 @@ -import './factory-tool-executor.live.test'; From cb1270701f5738b503f3354f798afdd6c6d2b035 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 18:38:00 -0400 Subject: [PATCH 07/13] Live tests fail-hard when harness not running, separate test:live script Live tests throw a clear error message when the realm server isn't running instead of silently skipping. They run via pnpm test:live (requires serve:support + cache:prepare + serve:realm), separate from pnpm test:node which doesn't need external services. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/software-factory/package.json | 1 + .../tests/factory-tool-executor.live.test.ts | 190 ++++++++---------- packages/software-factory/tests/index.ts | 1 - packages/software-factory/tests/live-index.ts | 1 + 4 files changed, 91 insertions(+), 102 deletions(-) create mode 100644 packages/software-factory/tests/live-index.ts diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index c64335dd371..9920f1ded2a 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -27,6 +27,7 @@ "test:node": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/test.ts --node-only", "test:playwright": "playwright test", "test:playwright:headed": "playwright test --headed", + "test:live": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/live-index.ts", "test:realm": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/run-realm-tests.ts" }, "devDependencies": { diff --git a/packages/software-factory/tests/factory-tool-executor.live.test.ts b/packages/software-factory/tests/factory-tool-executor.live.test.ts index e3f6389cdb1..c44bd5b161b 100644 --- a/packages/software-factory/tests/factory-tool-executor.live.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.live.test.ts @@ -13,7 +13,7 @@ * Auth uses the harness's known secret seed to mint JWTs directly, * matching the pattern in src/harness.ts — no Matrix login needed. * - * Tests skip gracefully when the realm server is not running. + * Tests FAIL with a clear message when the realm server is not running. */ import jwt from 'jsonwebtoken'; @@ -101,122 +101,110 @@ async function isRealmServerRunning(): Promise { // --------------------------------------------------------------------------- module('factory-tool-executor live', function (hooks) { - let serverRunning = false; - hooks.before(async function () { - serverRunning = await isRealmServerRunning(); - if (!serverRunning) { - console.log( - '\n [SKIP] Realm server not running at ' + - REALM_SERVER_URL + - ' — run: pnpm serve:support && pnpm cache:prepare && pnpm serve:realm\n', + let running = await isRealmServerRunning(); + if (!running) { + throw new Error( + `Realm server is not running at ${REALM_SERVER_URL}. ` + + `Start the harness first:\n` + + ` pnpm serve:support\n` + + ` pnpm cache:prepare\n` + + ` pnpm serve:realm`, ); } }); test('realm-read fetches .realm.json from the test realm', async function (assert) { - if (!serverRunning) { - assert.expect(0); - } else { - let realmJwt = buildRealmToken(TEST_REALM_URL); - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: realmJwt }), - ); - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-read', - toolArgs: { - 'realm-url': TEST_REALM_URL, - path: '.realm.json', - }, - }); + let realmJwt = buildRealmToken(TEST_REALM_URL); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ authorization: realmJwt }), + ); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': TEST_REALM_URL, + path: '.realm.json', + }, + }); - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); - assert.strictEqual(typeof result.output, 'object', 'output is an object'); - } + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + assert.strictEqual(typeof result.output, 'object', 'output is an object'); }); test('realm-search returns results from the test realm', async function (assert) { - if (!serverRunning) { - assert.expect(0); - } else { - let realmJwt = buildRealmToken(TEST_REALM_URL); - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: realmJwt }), - ); - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-search', - toolArgs: { - 'realm-url': TEST_REALM_URL, - query: JSON.stringify({ filter: {}, page: { size: 1 } }), - }, - }); + let realmJwt = buildRealmToken(TEST_REALM_URL); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ authorization: realmJwt }), + ); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': TEST_REALM_URL, + query: JSON.stringify({ filter: {}, page: { size: 1 } }), + }, + }); - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); - let output = result.output as { data?: unknown[] }; - assert.true(Array.isArray(output.data), 'output has data array'); - } + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + let output = result.output as { data?: unknown[] }; + assert.true(Array.isArray(output.data), 'output has data array'); }); test('realm-create creates a scratch realm with icon and background', async function (assert) { - if (!serverRunning) { - assert.expect(0); - } else { - let serverJwt = buildRealmServerToken(); - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: serverJwt }), - ); - - let timestamp = Date.now(); - let endpoint = `live-test-${timestamp}`; - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-create', - toolArgs: { - 'realm-server-url': REALM_SERVER_URL, - name: `Live Test ${timestamp}`, - endpoint, - }, - }); - - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); + let serverJwt = buildRealmServerToken(); + let registry = new ToolRegistry(); + let executor = new ToolExecutor( + registry, + makeExecutorConfig({ authorization: serverJwt }), + ); + + let timestamp = Date.now(); + let endpoint = `live-test-${timestamp}`; + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-create', + toolArgs: { + 'realm-server-url': REALM_SERVER_URL, + name: `Live Test ${timestamp}`, + endpoint, + }, + }); - let output = result.output as { - data?: { - type?: string; - id?: string; - attributes?: Record; - }; + assert.strictEqual( + result.exitCode, + 0, + `exitCode 0, got: ${JSON.stringify(result.output)}`, + ); + + let output = result.output as { + data?: { + type?: string; + id?: string; + attributes?: Record; }; - assert.strictEqual(output.data?.type, 'realm', 'response type is realm'); - assert.strictEqual( - typeof output.data?.id, - 'string', - 'response has realm id', - ); - } + }; + assert.strictEqual(output.data?.type, 'realm', 'response type is realm'); + assert.strictEqual( + typeof output.data?.id, + 'string', + 'response has realm id', + ); }); test('unregistered tool is rejected', async function (assert) { diff --git a/packages/software-factory/tests/index.ts b/packages/software-factory/tests/index.ts index 42cb123a847..298f2ac462a 100644 --- a/packages/software-factory/tests/index.ts +++ b/packages/software-factory/tests/index.ts @@ -6,6 +6,5 @@ import './factory-entrypoint.integration.test'; import './factory-target-realm.test'; import './factory-tool-executor.test'; import './factory-tool-executor.integration.test'; -import './factory-tool-executor.live.test'; import './factory-tool-registry.test'; import './realm-auth.test'; diff --git a/packages/software-factory/tests/live-index.ts b/packages/software-factory/tests/live-index.ts new file mode 100644 index 00000000000..a13f1d551c4 --- /dev/null +++ b/packages/software-factory/tests/live-index.ts @@ -0,0 +1 @@ +import './factory-tool-executor.live.test'; From fbb5f9cfe8196d5f927a8d2e951a30eee1b01381 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 18:39:48 -0400 Subject: [PATCH 08/13] Add test:live to CI workflow for software-factory Starts the harness (serve:support, cache:prepare, serve:realm) before running live integration tests in CI. Live tests fail-hard when the realm server isn't running with a clear error message. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9529bf8e96f..82faef44f7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -677,6 +677,26 @@ jobs: - name: Run Node tests run: pnpm test:node working-directory: packages/software-factory + - name: Start support services for live tests + uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 # 1.0.7 + with: + run: pnpm serve:support & + working-directory: packages/software-factory + wait-for: 3m + wait-on: file:///tmp/software-factory-runtime/support.json + - name: Prepare cache for live tests + run: pnpm cache:prepare + working-directory: packages/software-factory + - name: Start realm server for live tests + uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 # 1.0.7 + with: + run: pnpm serve:realm & + working-directory: packages/software-factory + wait-for: 3m + wait-on: http-get://localhost:4205/test/.realm.json + - name: Run live integration tests + run: pnpm test:live + working-directory: packages/software-factory - name: Serve host dist (test assets) uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 # 1.0.7 with: From 1fcdeee4c3da4b267b9ff76d70984b91835a000d Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 19:12:19 -0400 Subject: [PATCH 09/13] Fix live tests: use harness JWT seeds, remove realm-create (CS-10472) - realm-read and realm-search use REALM_SECRET_SEED for JWT minting - realm-create live test blocked by CS-10472 (orphaned process teardown) - Revert CI harness steps (process lifecycle not reliable yet) - Clean up unused REALM_SERVER_SECRET_SEED references Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yaml | 20 ----- .../tests/factory-tool-executor.live.test.ts | 89 ++++++------------- 2 files changed, 25 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 82faef44f7b..9529bf8e96f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -677,26 +677,6 @@ jobs: - name: Run Node tests run: pnpm test:node working-directory: packages/software-factory - - name: Start support services for live tests - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 # 1.0.7 - with: - run: pnpm serve:support & - working-directory: packages/software-factory - wait-for: 3m - wait-on: file:///tmp/software-factory-runtime/support.json - - name: Prepare cache for live tests - run: pnpm cache:prepare - working-directory: packages/software-factory - - name: Start realm server for live tests - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 # 1.0.7 - with: - run: pnpm serve:realm & - working-directory: packages/software-factory - wait-for: 3m - wait-on: http-get://localhost:4205/test/.realm.json - - name: Run live integration tests - run: pnpm test:live - working-directory: packages/software-factory - name: Serve host dist (test assets) uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 # 1.0.7 with: diff --git a/packages/software-factory/tests/factory-tool-executor.live.test.ts b/packages/software-factory/tests/factory-tool-executor.live.test.ts index c44bd5b161b..c6feead54ca 100644 --- a/packages/software-factory/tests/factory-tool-executor.live.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.live.test.ts @@ -36,7 +36,6 @@ const REALM_SERVER_PORT = Number( const REALM_SERVER_URL = `http://localhost:${REALM_SERVER_PORT}/`; const TEST_REALM_URL = `${REALM_SERVER_URL}test/`; const REALM_SECRET_SEED = "shhh! it's a secret"; -const REALM_SERVER_SECRET_SEED = "mum's the word"; const DEFAULT_REALM_OWNER = '@software-factory-owner:localhost'; // --------------------------------------------------------------------------- @@ -48,27 +47,19 @@ function buildRealmToken( user = DEFAULT_REALM_OWNER, permissions = ['read', 'write', 'realm-owner'], ): string { - return jwt.sign( - { - user, - realm: realmURL, - permissions, - sessionRoom: `software-factory-session-room-for-${user}`, - realmServerURL: REALM_SERVER_URL, - }, - REALM_SECRET_SEED, - { expiresIn: '7d' }, - ); -} - -function buildRealmServerToken(user = DEFAULT_REALM_OWNER): string { - return jwt.sign( - { - user, - sessionRoom: `software-factory-session-room-for-${user}`, - }, - REALM_SERVER_SECRET_SEED, - { expiresIn: '7d' }, + return ( + 'Bearer ' + + jwt.sign( + { + user, + realm: realmURL, + permissions, + sessionRoom: `software-factory-session-room-for-${user}`, + realmServerURL: REALM_SERVER_URL, + }, + REALM_SECRET_SEED, + { expiresIn: '7d' }, + ) ); } @@ -152,7 +143,15 @@ module('factory-tool-executor live', function (hooks) { tool: 'realm-search', toolArgs: { 'realm-url': TEST_REALM_URL, - query: JSON.stringify({ filter: {}, page: { size: 1 } }), + query: JSON.stringify({ + filter: { + type: { + module: 'https://cardstack.com/base/card-api', + name: 'CardDef', + }, + }, + page: { size: 1 }, + }), }, }); @@ -165,47 +164,9 @@ module('factory-tool-executor live', function (hooks) { assert.true(Array.isArray(output.data), 'output has data array'); }); - test('realm-create creates a scratch realm with icon and background', async function (assert) { - let serverJwt = buildRealmServerToken(); - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: serverJwt }), - ); - - let timestamp = Date.now(); - let endpoint = `live-test-${timestamp}`; - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-create', - toolArgs: { - 'realm-server-url': REALM_SERVER_URL, - name: `Live Test ${timestamp}`, - endpoint, - }, - }); - - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); - - let output = result.output as { - data?: { - type?: string; - id?: string; - attributes?: Record; - }; - }; - assert.strictEqual(output.data?.type, 'realm', 'response type is realm'); - assert.strictEqual( - typeof output.data?.id, - 'string', - 'response has realm id', - ); - }); + // realm-create live test is blocked by CS-10472 (harness process teardown + // leaves orphaned processes that interfere with subsequent realm creation). + // The request building and body shape are verified by unit + integration tests. test('unregistered tool is rejected', async function (assert) { let registry = new ToolRegistry(); From 3c97e4e7383d4a71b458433785486d0dbd14eeda Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 19:19:14 -0400 Subject: [PATCH 10/13] Document boxel-icons server prerequisite for cache:prepare The harness indexes cards that reference @cardstack/boxel-icons modules, so the icons server must be running on port 4206 before cache:prepare. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/software-factory/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/software-factory/README.md b/packages/software-factory/README.md index 2661103ee3d..882fe370cbd 100644 --- a/packages/software-factory/README.md +++ b/packages/software-factory/README.md @@ -18,6 +18,11 @@ the repo. Package linting currently runs `glint`, `eslint`, and `prettier`. - Docker running - Host app assets available at `http://localhost:4200/` - use `cd packages/host && pnpm serve:dist` +- Boxel icons server available at `http://localhost:4206/` + - use `cd packages/boxel-icons && pnpm serve` + - in a worktree where boxel-icons hasn't been built, symlink the dist from the main checkout: + `ln -s /path/to/boxel/packages/boxel-icons/dist packages/boxel-icons/dist` + - required before `cache:prepare` — the harness indexes cards that reference icon modules The harness starts its own seeded test Postgres, Synapse, prerender server, and isolated realm server. By default it serves the test realm and base realm from From b02753adce97e3f7b5d6eda7c6cbfad740496b46 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Sun, 22 Mar 2026 19:32:42 -0400 Subject: [PATCH 11/13] Address review: validate all URL args, fix SSRF gap, remove realm-reindex ref - Validate realm/realm-url args against allowed-realm list for ALL tool categories (not just realm-api), preventing SSRF and token exfiltration - Validate realm-server-url by origin match against allowed realm origins - Remove stale realm-reindex reference from validateDestructiveOps Co-Authored-By: Claude Opus 4.6 (1M context) --- .../scripts/lib/factory-tool-executor.ts | 67 ++++++++++++++++--- .../tests/factory-tool-executor.test.ts | 37 ++++++++++ 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/packages/software-factory/scripts/lib/factory-tool-executor.ts b/packages/software-factory/scripts/lib/factory-tool-executor.ts index 6487573771a..55b5373dc75 100644 --- a/packages/software-factory/scripts/lib/factory-tool-executor.ts +++ b/packages/software-factory/scripts/lib/factory-tool-executor.ts @@ -239,15 +239,22 @@ export class ToolExecutor { } } - // Allowed-realm targeting for realm-api tools (always enforced) - let manifest = this.registry.getManifest(toolName); - if (manifest?.category === 'realm-api') { - let realmUrl = toolArgs['realm-url']; - if (typeof realmUrl === 'string' && looksLikeUrl(realmUrl)) { - this.validateRealmTarget(toolName, realmUrl); + // Allowed-target validation for all URL args across all tool categories. + // Prevents SSRF and token exfiltration via the propagated Authorization header. + let urlArgNames = ['realm', 'realm-url']; + for (let argName of urlArgNames) { + let value = toolArgs[argName]; + if (typeof value === 'string' && looksLikeUrl(value)) { + this.validateRealmTarget(toolName, value); } } + // realm-server-url must match one of the allowed realm origins + let serverUrl = toolArgs['realm-server-url']; + if (typeof serverUrl === 'string' && looksLikeUrl(serverUrl)) { + this.validateRealmServerTarget(toolName, serverUrl); + } + // Extra validation for destructive operations this.validateDestructiveOps(toolName, toolArgs); } @@ -275,6 +282,47 @@ export class ToolExecutor { } } + /** + * Validate that a realm-server-url arg points to a server that hosts + * one of the allowed realms (origin match against target/test/prefixes). + */ + private validateRealmServerTarget(toolName: string, serverUrl: string): void { + let normalizedServer: string; + try { + normalizedServer = new URL(serverUrl).origin; + } catch { + throw new ToolSafetyError( + `Tool "${toolName}" has invalid realm-server-url: "${serverUrl}"`, + ); + } + + let allowedOrigins = new Set(); + try { + allowedOrigins.add(new URL(this.config.targetRealmUrl).origin); + } catch { + // skip invalid + } + try { + allowedOrigins.add(new URL(this.config.testRealmUrl).origin); + } catch { + // skip invalid + } + for (let prefix of this.config.allowedRealmPrefixes ?? []) { + try { + allowedOrigins.add(new URL(prefix).origin); + } catch { + // skip invalid + } + } + + if (!allowedOrigins.has(normalizedServer)) { + throw new ToolSafetyError( + `Tool "${toolName}" targets server "${serverUrl}" which is not in the allowed origins. ` + + `Allowed: ${[...allowedOrigins].join(', ')}`, + ); + } + } + private validateDestructiveOps( toolName: string, toolArgs: Record, @@ -302,11 +350,8 @@ export class ToolExecutor { } } - // realm-create and realm-reindex require extra validation - if (toolName === 'realm-create' || toolName === 'realm-reindex') { - // These are allowed but logged — the orchestrator trusts the agent - // chose them deliberately within the allowed realm set. - } + // realm-create is allowed but logged — the orchestrator trusts the agent + // chose it deliberately within the allowed realm set. } // ------------------------------------------------------------------------- diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts index 7da8fc5b7b5..2d22c7647a7 100644 --- a/packages/software-factory/tests/factory-tool-executor.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -236,6 +236,43 @@ module('factory-tool-executor > source realm protection', function () { assert.true((err as Error).message.includes('not in the allowed list')); } }); + + test('rejects realm-server-url targeting unknown origin', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute( + makeInvokeToolAction('realm-server-session', { + 'realm-server-url': 'https://evil.example.test/', + 'openid-token': 'token', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true( + (err as Error).message.includes('not in the allowed origins'), + ); + } + }); + + test('rejects script tool targeting unknown realm URL', async function (assert) { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, makeConfig()); + + try { + await executor.execute( + makeInvokeToolAction('search-realm', { + realm: 'https://evil.example.test/hacker/realm/', + }), + ); + assert.ok(false, 'should have thrown'); + } catch (err) { + assert.true(err instanceof ToolSafetyError); + assert.true((err as Error).message.includes('not in the allowed list')); + } + }); }); // --------------------------------------------------------------------------- From 966e5f8291b02f6479774daafe86fee051f74dd4 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 09:39:53 -0400 Subject: [PATCH 12/13] Move iconURLFor/getRandomBackgroundURL to @cardstack/runtime-common Extract realm icon and background URL functions into a shared module at runtime-common/realm-display-defaults.ts. Host app and software-factory now import from the same source, eliminating three duplicate copies. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/host/app/lib/utils.ts | 95 +-------------- .../runtime-common/realm-display-defaults.ts | 108 ++++++++++++++++++ .../scripts/lib/factory-tool-executor.ts | 108 +----------------- .../src/factory-target-realm.ts | 75 +----------- .../tests/factory-tool-executor.test.ts | 4 +- 5 files changed, 126 insertions(+), 264 deletions(-) create mode 100644 packages/runtime-common/realm-display-defaults.ts diff --git a/packages/host/app/lib/utils.ts b/packages/host/app/lib/utils.ts index 873e5c0ad73..cf0745e25e1 100644 --- a/packages/host/app/lib/utils.ts +++ b/packages/host/app/lib/utils.ts @@ -6,6 +6,10 @@ import { devSkillLocalPath, envSkillLocalPath, } from '@cardstack/runtime-common'; +export { + iconURLFor, + getRandomBackgroundURL, +} from '@cardstack/runtime-common/realm-display-defaults'; import ENV from '@cardstack/host/config/environment'; @@ -47,97 +51,6 @@ export function cleanseString(value: string) { .replace(/[^a-z0-9]$/, ''); } -export function iconURLFor(word: string) { - if (!word) { - return undefined; - } - let cleansedWord = cleanseString(word).replace(/^[0-9]+/, ''); - return iconURLs[cleansedWord.charAt(0)]; -} - -export function getRandomBackgroundURL() { - const index = Math.floor(Math.random() * backgroundURLs.length); - return backgroundURLs[index]; -} - -const iconURLs: { [letter: string]: string } = Object.freeze({ - a: 'https://boxel-images.boxel.ai/icons/Letter-a.png', - b: 'https://boxel-images.boxel.ai/icons/Letter-b.png', - c: 'https://boxel-images.boxel.ai/icons/Letter-c.png', - d: 'https://boxel-images.boxel.ai/icons/Letter-d.png', - e: 'https://boxel-images.boxel.ai/icons/Letter-e.png', - f: 'https://boxel-images.boxel.ai/icons/Letter-f.png', - g: 'https://boxel-images.boxel.ai/icons/Letter-g.png', - h: 'https://boxel-images.boxel.ai/icons/Letter-h.png', - i: 'https://boxel-images.boxel.ai/icons/Letter-i.png', - j: 'https://boxel-images.boxel.ai/icons/Letter-j.png', - k: 'https://boxel-images.boxel.ai/icons/Letter-k.png', - l: 'https://boxel-images.boxel.ai/icons/Letter-l.png', - m: 'https://boxel-images.boxel.ai/icons/Letter-m.png', - n: 'https://boxel-images.boxel.ai/icons/Letter-n.png', - o: 'https://boxel-images.boxel.ai/icons/Letter-o.png', - p: 'https://boxel-images.boxel.ai/icons/Letter-p.png', - q: 'https://boxel-images.boxel.ai/icons/Letter-q.png', - r: 'https://boxel-images.boxel.ai/icons/Letter-r.png', - s: 'https://boxel-images.boxel.ai/icons/Letter-s.png', - t: 'https://boxel-images.boxel.ai/icons/Letter-t.png', - u: 'https://boxel-images.boxel.ai/icons/Letter-u.png', - v: 'https://boxel-images.boxel.ai/icons/Letter-v.png', - w: 'https://boxel-images.boxel.ai/icons/Letter-w.png', - x: 'https://boxel-images.boxel.ai/icons/Letter-x.png', - y: 'https://boxel-images.boxel.ai/icons/Letter-y.png', - z: 'https://boxel-images.boxel.ai/icons/letter-z.png', -}); -const backgroundURLs: readonly string[] = Object.freeze([ - 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', -]); - export function urlForRealmLookup(card: CardDef) { let urlForRealmLookup = card.id ?? card[realmURL]?.href; if (!urlForRealmLookup) { diff --git a/packages/runtime-common/realm-display-defaults.ts b/packages/runtime-common/realm-display-defaults.ts new file mode 100644 index 00000000000..689fcef97df --- /dev/null +++ b/packages/runtime-common/realm-display-defaults.ts @@ -0,0 +1,108 @@ +/** + * Default icon and background URL generators for new realms. + * + * Shared by host, software-factory, and any other package that creates realms. + * Pure JS — no external dependencies. + */ + +const ICON_URLS: { [letter: string]: string } = Object.freeze({ + a: 'https://boxel-images.boxel.ai/icons/Letter-a.png', + b: 'https://boxel-images.boxel.ai/icons/Letter-b.png', + c: 'https://boxel-images.boxel.ai/icons/Letter-c.png', + d: 'https://boxel-images.boxel.ai/icons/Letter-d.png', + e: 'https://boxel-images.boxel.ai/icons/Letter-e.png', + f: 'https://boxel-images.boxel.ai/icons/Letter-f.png', + g: 'https://boxel-images.boxel.ai/icons/Letter-g.png', + h: 'https://boxel-images.boxel.ai/icons/Letter-h.png', + i: 'https://boxel-images.boxel.ai/icons/Letter-i.png', + j: 'https://boxel-images.boxel.ai/icons/Letter-j.png', + k: 'https://boxel-images.boxel.ai/icons/Letter-k.png', + l: 'https://boxel-images.boxel.ai/icons/Letter-l.png', + m: 'https://boxel-images.boxel.ai/icons/Letter-m.png', + n: 'https://boxel-images.boxel.ai/icons/Letter-n.png', + o: 'https://boxel-images.boxel.ai/icons/Letter-o.png', + p: 'https://boxel-images.boxel.ai/icons/Letter-p.png', + q: 'https://boxel-images.boxel.ai/icons/Letter-q.png', + r: 'https://boxel-images.boxel.ai/icons/Letter-r.png', + s: 'https://boxel-images.boxel.ai/icons/Letter-s.png', + t: 'https://boxel-images.boxel.ai/icons/Letter-t.png', + u: 'https://boxel-images.boxel.ai/icons/Letter-u.png', + v: 'https://boxel-images.boxel.ai/icons/Letter-v.png', + w: 'https://boxel-images.boxel.ai/icons/Letter-w.png', + x: 'https://boxel-images.boxel.ai/icons/Letter-x.png', + y: 'https://boxel-images.boxel.ai/icons/Letter-y.png', + z: 'https://boxel-images.boxel.ai/icons/letter-z.png', +}); + +const BACKGROUND_URLS: readonly string[] = Object.freeze([ + 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', + 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', +]); + +/** + * Generate a letter-based icon URL from a realm/workspace name. + * Returns undefined if the name has no alphabetic characters. + */ +export function iconURLFor(word: string): string | undefined { + if (!word) { + return undefined; + } + let cleansed = word + .toLowerCase() + .replace(/[^a-z0-9]/g, '') + .replace(/^[0-9]+/, ''); + return ICON_URLS[cleansed.charAt(0)]; +} + +/** + * Pick a random background image URL from the predefined set. + */ +export function getRandomBackgroundURL(): string { + let index = Math.floor(Math.random() * BACKGROUND_URLS.length); + return BACKGROUND_URLS[index]; +} diff --git a/packages/software-factory/scripts/lib/factory-tool-executor.ts b/packages/software-factory/scripts/lib/factory-tool-executor.ts index 55b5373dc75..9bfa9087979 100644 --- a/packages/software-factory/scripts/lib/factory-tool-executor.ts +++ b/packages/software-factory/scripts/lib/factory-tool-executor.ts @@ -1,6 +1,11 @@ import { spawn } from 'node:child_process'; import { resolve } from 'node:path'; +import { + iconURLFor, + getRandomBackgroundURL, +} from '@cardstack/runtime-common/realm-display-defaults'; + import type { AgentAction, ToolResult } from './factory-agent'; import type { ToolRegistry } from './factory-tool-registry'; @@ -839,7 +844,7 @@ function buildRealmApiRequest( let iconURL = typeof toolArgs['iconURL'] === 'string' ? toolArgs['iconURL'] - : iconURLForName(name); + : iconURLFor(name); let backgroundURL = typeof toolArgs['backgroundURL'] === 'string' ? toolArgs['backgroundURL'] @@ -919,104 +924,3 @@ function extractCreatedRealmUrl(responseBody: unknown): string | undefined { } return undefined; } - -// --------------------------------------------------------------------------- -// Icon and background URL defaults -// TODO: Move iconURLForName / getRandomBackgroundURL to @cardstack/runtime-common -// so host (packages/host/app/lib/utils.ts) and software-factory share one copy. -// --------------------------------------------------------------------------- - -const ICON_URLS: Record = { - a: 'https://boxel-images.boxel.ai/icons/Letter-a.png', - b: 'https://boxel-images.boxel.ai/icons/Letter-b.png', - c: 'https://boxel-images.boxel.ai/icons/Letter-c.png', - d: 'https://boxel-images.boxel.ai/icons/Letter-d.png', - e: 'https://boxel-images.boxel.ai/icons/Letter-e.png', - f: 'https://boxel-images.boxel.ai/icons/Letter-f.png', - g: 'https://boxel-images.boxel.ai/icons/Letter-g.png', - h: 'https://boxel-images.boxel.ai/icons/Letter-h.png', - i: 'https://boxel-images.boxel.ai/icons/Letter-i.png', - j: 'https://boxel-images.boxel.ai/icons/Letter-j.png', - k: 'https://boxel-images.boxel.ai/icons/Letter-k.png', - l: 'https://boxel-images.boxel.ai/icons/Letter-l.png', - m: 'https://boxel-images.boxel.ai/icons/Letter-m.png', - n: 'https://boxel-images.boxel.ai/icons/Letter-n.png', - o: 'https://boxel-images.boxel.ai/icons/Letter-o.png', - p: 'https://boxel-images.boxel.ai/icons/Letter-p.png', - q: 'https://boxel-images.boxel.ai/icons/Letter-q.png', - r: 'https://boxel-images.boxel.ai/icons/Letter-r.png', - s: 'https://boxel-images.boxel.ai/icons/Letter-s.png', - t: 'https://boxel-images.boxel.ai/icons/Letter-t.png', - u: 'https://boxel-images.boxel.ai/icons/Letter-u.png', - v: 'https://boxel-images.boxel.ai/icons/Letter-v.png', - w: 'https://boxel-images.boxel.ai/icons/Letter-w.png', - x: 'https://boxel-images.boxel.ai/icons/Letter-x.png', - y: 'https://boxel-images.boxel.ai/icons/Letter-y.png', - z: 'https://boxel-images.boxel.ai/icons/letter-z.png', -}; - -const BACKGROUND_URLS: readonly string[] = [ - 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', -]; - -export function iconURLForName(name: string): string | undefined { - if (!name) { - return undefined; - } - let cleansed = name - .toLowerCase() - .replace(/[^a-z0-9]/g, '') - .replace(/^[0-9]+/, ''); - return ICON_URLS[cleansed.charAt(0)]; -} - -export function getRandomBackgroundURL(): string { - let index = Math.floor(Math.random() * BACKGROUND_URLS.length); - return BACKGROUND_URLS[index]; -} diff --git a/packages/software-factory/src/factory-target-realm.ts b/packages/software-factory/src/factory-target-realm.ts index 31a57837ccb..ba6480d60f2 100644 --- a/packages/software-factory/src/factory-target-realm.ts +++ b/packages/software-factory/src/factory-target-realm.ts @@ -1,6 +1,10 @@ import { getMatrixUsername } from '@cardstack/runtime-common/matrix-client'; import { APP_BOXEL_REALMS_EVENT_TYPE } from '@cardstack/runtime-common/matrix-constants'; import { ensureTrailingSlash } from '@cardstack/runtime-common/paths'; +import { + iconURLFor, + getRandomBackgroundURL, +} from '@cardstack/runtime-common/realm-display-defaults'; import { SupportedMimeType } from '@cardstack/runtime-common/router'; import { @@ -105,8 +109,8 @@ async function createRealm( attributes: { endpoint, name: endpoint, - iconURL: iconURLForRealmName(endpoint), - backgroundURL: randomBackgroundURL(), + iconURL: iconURLFor(endpoint), + backgroundURL: getRandomBackgroundURL(), }, }, }), @@ -377,70 +381,3 @@ function normalizeOptionalString( let trimmed = value.trim(); return trimmed === '' ? undefined : trimmed; } - -// Mirrors iconURLFor() from packages/host/app/lib/utils.ts -function iconURLForRealmName(name: string): string { - let letter = name - .toLowerCase() - .replace(/[^a-z]/g, '') - .charAt(0); - if (!letter) { - letter = 'b'; // fallback for names starting with numbers/symbols - } - return `https://boxel-images.boxel.ai/icons/Letter-${letter}.png`; -} - -// Mirrors getRandomBackgroundURL() from packages/host/app/lib/utils.ts -const backgroundURLs = [ - 'https://boxel-images.boxel.ai/background-images/4k-arabic-teal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-arrow-weave.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-atmosphere-curvature.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-brushed-slabs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-coral-reefs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-crescent-lake.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-curvilinear-stairs.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-desert-dunes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-doodle-board.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-fallen-leaves.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-flowing-mesh.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glass-reflection.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-glow-cells.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-granite-peaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-green-wormhole.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-joshua-dawn.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lava-river.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-leaves-moss.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-light-streaks.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-lowres-glitch.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-marble-shimmer.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-metallic-leather.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-microscopic-crystals.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-moon-face.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-mountain-runway.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-origami-flock.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-paint-swirl.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-pastel-triangles.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-perforated-sheet.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-plastic-ripples.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-powder-puff.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-radiant-crystal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-redrock-canyon.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rock-portal.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-rolling-hills.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-sand-stone.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-silver-fur.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-spa-pool.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stained-glass.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-stone-veins.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-tangerine-plains.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-techno-floor.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-thick-frost.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-water-surface.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-watercolor-splashes.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wildflower-field.jpg', - 'https://boxel-images.boxel.ai/background-images/4k-wood-grain.jpg', -] as const; - -function randomBackgroundURL(): string { - return backgroundURLs[Math.floor(Math.random() * backgroundURLs.length)]; -} diff --git a/packages/software-factory/tests/factory-tool-executor.test.ts b/packages/software-factory/tests/factory-tool-executor.test.ts index 2d22c7647a7..8c2dc8da90b 100644 --- a/packages/software-factory/tests/factory-tool-executor.test.ts +++ b/packages/software-factory/tests/factory-tool-executor.test.ts @@ -6,10 +6,10 @@ import { ToolNotFoundError, ToolSafetyError, ToolTimeoutError, - iconURLForName, type ToolExecutionLogEntry, type ToolExecutorConfig, } from '../scripts/lib/factory-tool-executor'; +import { iconURLFor } from '@cardstack/runtime-common/realm-display-defaults'; import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; // --------------------------------------------------------------------------- @@ -639,7 +639,7 @@ module('factory-tool-executor > realm-api execution', function () { let body = JSON.parse(capturedBody!); assert.strictEqual( body.data.attributes.iconURL, - iconURLForName('My Realm'), + iconURLFor('My Realm'), 'iconURL defaults from name', ); }); From 74a82545f651c936fcfabcfa059505f788fdfbda Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Tue, 24 Mar 2026 09:47:09 -0400 Subject: [PATCH 13/13] Move live tests to Playwright spec, remove test:live script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live tool executor tests now run as a Playwright spec (factory-tool-executor.spec.ts) alongside the other browser specs, sharing the harness lifecycle. Removes the separate test:live script and QUnit live test file — no more silent skipping. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/software-factory/package.json | 1 - .../tests/factory-tool-executor.live.test.ts | 186 ------------------ .../tests/factory-tool-executor.spec.ts | 95 +++++++++ packages/software-factory/tests/live-index.ts | 1 - 4 files changed, 95 insertions(+), 188 deletions(-) delete mode 100644 packages/software-factory/tests/factory-tool-executor.live.test.ts create mode 100644 packages/software-factory/tests/factory-tool-executor.spec.ts delete mode 100644 packages/software-factory/tests/live-index.ts diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index 9920f1ded2a..c64335dd371 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -27,7 +27,6 @@ "test:node": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/test.ts --node-only", "test:playwright": "playwright test", "test:playwright:headed": "playwright test --headed", - "test:live": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/live-index.ts", "test:realm": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/run-realm-tests.ts" }, "devDependencies": { diff --git a/packages/software-factory/tests/factory-tool-executor.live.test.ts b/packages/software-factory/tests/factory-tool-executor.live.test.ts deleted file mode 100644 index c6feead54ca..00000000000 --- a/packages/software-factory/tests/factory-tool-executor.live.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Live integration tests for the ToolExecutor against the software-factory - * harness realm server. - * - * These tests hit the real realm server APIs to verify the tool executor - * produces requests the server accepts and responses match expected shapes. - * - * Prerequisites: - * 1. pnpm serve:support # starts Matrix, Postgres, prerender - * 2. pnpm cache:prepare # creates template database - * 3. pnpm serve:realm # starts realm server on port 4205 - * - * Auth uses the harness's known secret seed to mint JWTs directly, - * matching the pattern in src/harness.ts — no Matrix login needed. - * - * Tests FAIL with a clear message when the realm server is not running. - */ - -import jwt from 'jsonwebtoken'; -import { module, test } from 'qunit'; - -import { - ToolExecutor, - ToolNotFoundError, - type ToolExecutorConfig, -} from '../scripts/lib/factory-tool-executor'; -import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; - -// --------------------------------------------------------------------------- -// Harness constants (match src/harness.ts) -// --------------------------------------------------------------------------- - -const REALM_SERVER_PORT = Number( - process.env.SOFTWARE_FACTORY_REALM_PORT ?? 4205, -); -const REALM_SERVER_URL = `http://localhost:${REALM_SERVER_PORT}/`; -const TEST_REALM_URL = `${REALM_SERVER_URL}test/`; -const REALM_SECRET_SEED = "shhh! it's a secret"; -const DEFAULT_REALM_OWNER = '@software-factory-owner:localhost'; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function buildRealmToken( - realmURL: string, - user = DEFAULT_REALM_OWNER, - permissions = ['read', 'write', 'realm-owner'], -): string { - return ( - 'Bearer ' + - jwt.sign( - { - user, - realm: realmURL, - permissions, - sessionRoom: `software-factory-session-room-for-${user}`, - realmServerURL: REALM_SERVER_URL, - }, - REALM_SECRET_SEED, - { expiresIn: '7d' }, - ) - ); -} - -function makeExecutorConfig( - overrides?: Partial, -): ToolExecutorConfig { - return { - packageRoot: process.cwd(), - targetRealmUrl: TEST_REALM_URL, - testRealmUrl: TEST_REALM_URL, - allowedRealmPrefixes: [REALM_SERVER_URL], - ...overrides, - }; -} - -async function isRealmServerRunning(): Promise { - try { - let response = await fetch(REALM_SERVER_URL, { - method: 'HEAD', - signal: AbortSignal.timeout(2000), - }); - return response.ok || response.status === 403 || response.status === 404; - } catch { - return false; - } -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -module('factory-tool-executor live', function (hooks) { - hooks.before(async function () { - let running = await isRealmServerRunning(); - if (!running) { - throw new Error( - `Realm server is not running at ${REALM_SERVER_URL}. ` + - `Start the harness first:\n` + - ` pnpm serve:support\n` + - ` pnpm cache:prepare\n` + - ` pnpm serve:realm`, - ); - } - }); - - test('realm-read fetches .realm.json from the test realm', async function (assert) { - let realmJwt = buildRealmToken(TEST_REALM_URL); - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: realmJwt }), - ); - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-read', - toolArgs: { - 'realm-url': TEST_REALM_URL, - path: '.realm.json', - }, - }); - - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); - assert.strictEqual(typeof result.output, 'object', 'output is an object'); - }); - - test('realm-search returns results from the test realm', async function (assert) { - let realmJwt = buildRealmToken(TEST_REALM_URL); - let registry = new ToolRegistry(); - let executor = new ToolExecutor( - registry, - makeExecutorConfig({ authorization: realmJwt }), - ); - - let result = await executor.execute({ - type: 'invoke_tool', - tool: 'realm-search', - toolArgs: { - 'realm-url': TEST_REALM_URL, - query: JSON.stringify({ - filter: { - type: { - module: 'https://cardstack.com/base/card-api', - name: 'CardDef', - }, - }, - page: { size: 1 }, - }), - }, - }); - - assert.strictEqual( - result.exitCode, - 0, - `exitCode 0, got: ${JSON.stringify(result.output)}`, - ); - let output = result.output as { data?: unknown[] }; - assert.true(Array.isArray(output.data), 'output has data array'); - }); - - // realm-create live test is blocked by CS-10472 (harness process teardown - // leaves orphaned processes that interfere with subsequent realm creation). - // The request building and body shape are verified by unit + integration tests. - - test('unregistered tool is rejected', async function (assert) { - let registry = new ToolRegistry(); - let executor = new ToolExecutor(registry, makeExecutorConfig()); - - try { - await executor.execute({ - type: 'invoke_tool', - tool: 'shell-exec-arbitrary', - toolArgs: { command: 'rm -rf /' }, - }); - assert.ok(false, 'should have thrown'); - } catch (err) { - assert.true(err instanceof ToolNotFoundError, 'throws ToolNotFoundError'); - } - }); -}); diff --git a/packages/software-factory/tests/factory-tool-executor.spec.ts b/packages/software-factory/tests/factory-tool-executor.spec.ts new file mode 100644 index 00000000000..e58fdf11512 --- /dev/null +++ b/packages/software-factory/tests/factory-tool-executor.spec.ts @@ -0,0 +1,95 @@ +/** + * Live integration tests for the ToolExecutor against the software-factory + * harness realm server. + * + * These run as Playwright specs so they share the harness lifecycle + * (global-setup starts serve:support + cache:prepare, fixtures start + * serve:realm per spec). No browser is needed — these are pure Node tests + * that happen to use the Playwright test runner for harness management. + */ + +import { test } from './fixtures'; +import { expect } from '@playwright/test'; + +import { + ToolExecutor, + ToolNotFoundError, +} from '../scripts/lib/factory-tool-executor'; +import { ToolRegistry } from '../scripts/lib/factory-tool-registry'; + +test('realm-read fetches .realm.json from the test realm', async ({ + realm, +}) => { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: realm.realmURL.href, + testRealmUrl: realm.realmURL.href, + allowedRealmPrefixes: [realm.realmURL.origin + '/'], + authorization: `Bearer ${realm.ownerBearerToken}`, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-read', + toolArgs: { + 'realm-url': realm.realmURL.href, + path: '.realm.json', + }, + }); + + expect(result.exitCode).toBe(0); + expect(typeof result.output).toBe('object'); +}); + +test('realm-search returns results from the test realm', async ({ realm }) => { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: realm.realmURL.href, + testRealmUrl: realm.realmURL.href, + allowedRealmPrefixes: [realm.realmURL.origin + '/'], + authorization: `Bearer ${realm.ownerBearerToken}`, + }); + + let result = await executor.execute({ + type: 'invoke_tool', + tool: 'realm-search', + toolArgs: { + 'realm-url': realm.realmURL.href, + query: JSON.stringify({ + filter: { + type: { + module: 'https://cardstack.com/base/card-api', + name: 'CardDef', + }, + }, + page: { size: 1 }, + }), + }, + }); + + expect(result.exitCode).toBe(0); + let output = result.output as { data?: unknown[] }; + expect(Array.isArray(output.data)).toBe(true); +}); + +test('unregistered tool is rejected without reaching the server', async ({ + realm, +}) => { + let registry = new ToolRegistry(); + let executor = new ToolExecutor(registry, { + packageRoot: process.cwd(), + targetRealmUrl: realm.realmURL.href, + testRealmUrl: realm.realmURL.href, + authorization: `Bearer ${realm.ownerBearerToken}`, + }); + + await expect( + executor.execute({ + type: 'invoke_tool', + tool: 'shell-exec-arbitrary', + toolArgs: { command: 'rm -rf /' }, + }), + ).rejects.toThrow(ToolNotFoundError); +}); diff --git a/packages/software-factory/tests/live-index.ts b/packages/software-factory/tests/live-index.ts deleted file mode 100644 index a13f1d551c4..00000000000 --- a/packages/software-factory/tests/live-index.ts +++ /dev/null @@ -1 +0,0 @@ -import './factory-tool-executor.live.test';