From 8655eaa818b875aadaea9fcae3b93703fb9a46dc Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 7 Feb 2026 12:16:07 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=85=20test:=20add=20MCP=20smoke-test?= =?UTF-8?q?=20infrastructure=20and=20initial=20e2e=20test=20suites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce an in-process MCP test harness that boots a real server with InMemoryTransport, injects a capturing command executor, and exposes an MCP client for end-to-end assertions. Includes four smoke-test suites: CLI surface (help/version/tools), MCP discovery (listTools coverage), MCP invocation (callTool with mocked commands), and session management (defaults round-trip). Adds test-only reset/override hooks in command.ts, server-state.ts, tool-registry.ts, and debugger/tool-context.ts to support singleton teardown between test runs. Separates smoke tests from unit tests via a dedicated vitest.smoke.config.ts and a new test:smoke npm script. --- package.json | 1 + .../__tests__/debugging-tools.test.ts | 887 ++++++++++++++++++ src/server/server-state.ts | 4 + src/smoke-tests/__tests__/cli-surface.test.ts | 129 +++ .../__tests__/e2e-mcp-device-macos.test.ts | 455 +++++++++ .../__tests__/e2e-mcp-discovery.test.ts | 199 ++++ .../__tests__/e2e-mcp-doctor.test.ts | 38 + .../__tests__/e2e-mcp-error-paths.test.ts | 193 ++++ .../__tests__/e2e-mcp-invocation.test.ts | 327 +++++++ .../__tests__/e2e-mcp-logging.test.ts | 119 +++ .../__tests__/e2e-mcp-scaffolding.test.ts | 48 + .../__tests__/e2e-mcp-sessions.test.ts | 188 ++++ .../__tests__/e2e-mcp-simulator.test.ts | 369 ++++++++ .../__tests__/e2e-mcp-swift-package.test.ts | 114 +++ .../__tests__/e2e-mcp-ui-automation.test.ts | 420 +++++++++ src/smoke-tests/mcp-test-harness.ts | 215 +++++ src/utils/command.ts | 18 + src/utils/debugger/index.ts | 6 +- src/utils/debugger/tool-context.ts | 13 + src/utils/execution/index.ts | 8 +- src/utils/tool-registry.ts | 9 + vitest.config.ts | 1 + vitest.smoke.config.ts | 28 + 23 files changed, 3787 insertions(+), 2 deletions(-) create mode 100644 src/mcp/tools/debugging/__tests__/debugging-tools.test.ts create mode 100644 src/smoke-tests/__tests__/cli-surface.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-discovery.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-logging.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts create mode 100644 src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts create mode 100644 src/smoke-tests/mcp-test-harness.ts create mode 100644 vitest.smoke.config.ts diff --git a/package.json b/package.json index 66f8c782..93bdd6d7 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "docs:update": "npx tsx scripts/update-tools-docs.ts", "docs:update:dry-run": "npx tsx scripts/update-tools-docs.ts --dry-run --verbose", "test": "vitest run", + "test:smoke": "vitest run --config vitest.smoke.config.ts", "test:watch": "vitest", "test:ui": "vitest --ui", "test:coverage": "vitest run --coverage" diff --git a/src/mcp/tools/debugging/__tests__/debugging-tools.test.ts b/src/mcp/tools/debugging/__tests__/debugging-tools.test.ts new file mode 100644 index 00000000..e4e34eeb --- /dev/null +++ b/src/mcp/tools/debugging/__tests__/debugging-tools.test.ts @@ -0,0 +1,887 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { createMockExecutor } from '../../../../test-utils/mock-executors.ts'; +import { sessionStore } from '../../../../utils/session-store.ts'; +import { DebuggerManager } from '../../../../utils/debugger/index.ts'; +import type { DebuggerToolContext } from '../../../../utils/debugger/index.ts'; +import type { DebuggerBackend } from '../../../../utils/debugger/backends/DebuggerBackend.ts'; +import type { BreakpointSpec, DebugSessionInfo } from '../../../../utils/debugger/types.ts'; + +import { + schema as attachSchema, + handler as attachHandler, + debug_attach_simLogic, +} from '../debug_attach_sim.ts'; +import { + schema as bpAddSchema, + handler as bpAddHandler, + debug_breakpoint_addLogic, +} from '../debug_breakpoint_add.ts'; +import { + schema as bpRemoveSchema, + handler as bpRemoveHandler, + debug_breakpoint_removeLogic, +} from '../debug_breakpoint_remove.ts'; +import { + schema as continueSchema, + handler as continueHandler, + debug_continueLogic, +} from '../debug_continue.ts'; +import { + schema as detachSchema, + handler as detachHandler, + debug_detachLogic, +} from '../debug_detach.ts'; +import { + schema as lldbSchema, + handler as lldbHandler, + debug_lldb_commandLogic, +} from '../debug_lldb_command.ts'; +import { + schema as stackSchema, + handler as stackHandler, + debug_stackLogic, +} from '../debug_stack.ts'; +import { + schema as variablesSchema, + handler as variablesHandler, + debug_variablesLogic, +} from '../debug_variables.ts'; + +function createMockBackend(overrides: Partial = {}): DebuggerBackend { + return { + kind: 'dap', + attach: async () => {}, + detach: async () => {}, + runCommand: async () => 'mock output', + resume: async () => {}, + addBreakpoint: async (spec: BreakpointSpec) => ({ + id: 1, + spec, + rawOutput: 'Breakpoint 1: mock', + }), + removeBreakpoint: async () => 'removed', + getStack: async () => 'frame #0: mock stack', + getVariables: async () => 'x = 42', + getExecutionState: async () => ({ status: 'stopped' as const }), + dispose: async () => {}, + ...overrides, + }; +} + +function createTestDebuggerManager( + backendOverrides: Partial = {}, +): DebuggerManager { + const backend = createMockBackend(backendOverrides); + return new DebuggerManager({ + backendFactory: async () => backend, + }); +} + +function createTestContext(backendOverrides: Partial = {}): DebuggerToolContext { + return { + executor: createMockExecutor({ success: true, output: '' }), + debugger: createTestDebuggerManager(backendOverrides), + }; +} + +async function createSessionAndContext( + backendOverrides: Partial = {}, +): Promise<{ ctx: DebuggerToolContext; session: DebugSessionInfo }> { + const ctx = createTestContext(backendOverrides); + const session = await ctx.debugger.createSession({ + simulatorId: 'test-sim-uuid', + pid: 1234, + }); + ctx.debugger.setCurrentSession(session.id); + return { ctx, session }; +} + +// --------------------------------------------------------------------------- +// debug_attach_sim +// --------------------------------------------------------------------------- +describe('debug_attach_sim', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof attachHandler).toBe('function'); + }); + + it('should expose schema with expected shape', () => { + expect(attachSchema).toBeDefined(); + }); + }); + + describe('Handler Requirements', () => { + it('should return error when no session defaults for simulator', async () => { + const result = await attachHandler({ + bundleId: 'com.test.app', + }); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('simulatorId'); + }); + }); + + describe('Logic Behavior', () => { + it('should attach successfully with pid', async () => { + const ctx = createTestContext(); + + const result = await debug_attach_simLogic( + { + simulatorId: 'test-sim-uuid', + pid: 1234, + continueOnAttach: true, + makeCurrent: true, + }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Attached'); + expect(text).toContain('1234'); + expect(text).toContain('test-sim-uuid'); + expect(text).toContain('Debug session ID:'); + }); + + it('should attach without continuing when continueOnAttach is false', async () => { + const ctx = createTestContext(); + + const result = await debug_attach_simLogic( + { + simulatorId: 'test-sim-uuid', + pid: 1234, + continueOnAttach: false, + makeCurrent: true, + }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Execution is paused'); + }); + + it('should return error when createSession throws', async () => { + const ctx = createTestContext({ + attach: async () => { + throw new Error('LLDB attach failed'); + }, + }); + + const result = await debug_attach_simLogic( + { + simulatorId: 'test-sim-uuid', + pid: 1234, + continueOnAttach: true, + makeCurrent: true, + }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to attach debugger'); + expect(text).toContain('LLDB attach failed'); + }); + + it('should return error when resume throws after attach', async () => { + const ctx = createTestContext({ + resume: async () => { + throw new Error('Resume failed'); + }, + }); + + const result = await debug_attach_simLogic( + { + simulatorId: 'test-sim-uuid', + pid: 1234, + continueOnAttach: true, + makeCurrent: true, + }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to resume debugger after attach'); + }); + + it('should return error when simulator resolution fails', async () => { + const ctx: DebuggerToolContext = { + executor: createMockExecutor({ + success: false, + error: 'No simulators found', + }), + debugger: createTestDebuggerManager(), + }; + + const result = await debug_attach_simLogic( + { + simulatorName: 'NonExistent Simulator', + bundleId: 'com.test.app', + continueOnAttach: true, + makeCurrent: true, + }, + ctx, + ); + + expect(result.isError).toBe(true); + }); + + it('should return error when pid resolution fails for bundleId', async () => { + const ctx: DebuggerToolContext = { + executor: createMockExecutor({ + success: false, + error: 'launchctl failed', + }), + debugger: createTestDebuggerManager(), + }; + + const result = await debug_attach_simLogic( + { + simulatorId: 'test-sim-uuid', + bundleId: 'com.test.app', + continueOnAttach: true, + makeCurrent: true, + }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to resolve simulator PID'); + }); + + it('should include nextSteps on success', async () => { + const ctx = createTestContext(); + + const result = await debug_attach_simLogic( + { + simulatorId: 'test-sim-uuid', + pid: 1234, + continueOnAttach: true, + makeCurrent: true, + }, + ctx, + ); + + expect(result.nextSteps).toBeDefined(); + expect(result.nextSteps!.length).toBeGreaterThan(0); + const tools = result.nextSteps!.map((s) => s.tool); + expect(tools).toContain('debug_breakpoint_add'); + expect(tools).toContain('debug_continue'); + expect(tools).toContain('debug_stack'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_breakpoint_add +// --------------------------------------------------------------------------- +describe('debug_breakpoint_add', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof bpAddHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(bpAddSchema).toBeDefined(); + expect('debugSessionId' in bpAddSchema).toBe(true); + expect('file' in bpAddSchema).toBe(true); + expect('line' in bpAddSchema).toBe(true); + expect('function' in bpAddSchema).toBe(true); + expect('condition' in bpAddSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_breakpoint_addLogic({ file: 'main.swift', line: 10 }, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should add file-line breakpoint successfully', async () => { + const { ctx, session } = await createSessionAndContext(); + + const result = await debug_breakpoint_addLogic( + { debugSessionId: session.id, file: 'main.swift', line: 42 }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Breakpoint'); + expect(text).toContain('set'); + }); + + it('should add function breakpoint successfully', async () => { + const { ctx, session } = await createSessionAndContext(); + + const result = await debug_breakpoint_addLogic( + { debugSessionId: session.id, function: 'viewDidLoad' }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Breakpoint'); + }); + + it('should add breakpoint with condition', async () => { + const { ctx, session } = await createSessionAndContext(); + + const result = await debug_breakpoint_addLogic( + { + debugSessionId: session.id, + file: 'main.swift', + line: 10, + condition: 'x > 5', + }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Breakpoint'); + }); + + it('should return error when addBreakpoint throws', async () => { + const { ctx, session } = await createSessionAndContext({ + addBreakpoint: async () => { + throw new Error('Invalid file path'); + }, + }); + + const result = await debug_breakpoint_addLogic( + { debugSessionId: session.id, file: 'missing.swift', line: 1 }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to add breakpoint'); + expect(text).toContain('Invalid file path'); + }); + + it('should use current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext(); + + const result = await debug_breakpoint_addLogic({ file: 'main.swift', line: 10 }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Breakpoint'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_breakpoint_remove +// --------------------------------------------------------------------------- +describe('debug_breakpoint_remove', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof bpRemoveHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(bpRemoveSchema).toBeDefined(); + expect('debugSessionId' in bpRemoveSchema).toBe(true); + expect('breakpointId' in bpRemoveSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_breakpoint_removeLogic({ breakpointId: 1 }, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should remove breakpoint successfully', async () => { + const { ctx, session } = await createSessionAndContext(); + + const result = await debug_breakpoint_removeLogic( + { debugSessionId: session.id, breakpointId: 1 }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Breakpoint 1 removed'); + }); + + it('should return error when removeBreakpoint throws', async () => { + const { ctx, session } = await createSessionAndContext({ + removeBreakpoint: async () => { + throw new Error('Breakpoint not found'); + }, + }); + + const result = await debug_breakpoint_removeLogic( + { debugSessionId: session.id, breakpointId: 999 }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to remove breakpoint'); + expect(text).toContain('Breakpoint not found'); + }); + + it('should use current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext(); + + const result = await debug_breakpoint_removeLogic({ breakpointId: 1 }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Breakpoint 1 removed'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_continue +// --------------------------------------------------------------------------- +describe('debug_continue', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof continueHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(continueSchema).toBeDefined(); + expect('debugSessionId' in continueSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_continueLogic({}, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should resume session successfully with explicit id', async () => { + const { ctx, session } = await createSessionAndContext(); + + const result = await debug_continueLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Resumed debugger session'); + expect(text).toContain(session.id); + }); + + it('should resume current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext(); + + const result = await debug_continueLogic({}, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Resumed debugger session'); + }); + + it('should return error when resume throws', async () => { + const { ctx, session } = await createSessionAndContext({ + resume: async () => { + throw new Error('Process terminated'); + }, + }); + + const result = await debug_continueLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to resume debugger'); + expect(text).toContain('Process terminated'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_detach +// --------------------------------------------------------------------------- +describe('debug_detach', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof detachHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(detachSchema).toBeDefined(); + expect('debugSessionId' in detachSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_detachLogic({}, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should detach session successfully with explicit id', async () => { + const { ctx, session } = await createSessionAndContext(); + + const result = await debug_detachLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Detached debugger session'); + expect(text).toContain(session.id); + }); + + it('should detach current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext(); + + const result = await debug_detachLogic({}, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toContain('Detached debugger session'); + }); + + it('should return error when detach throws', async () => { + const { ctx, session } = await createSessionAndContext({ + detach: async () => { + throw new Error('Connection lost'); + }, + }); + + const result = await debug_detachLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to detach debugger'); + expect(text).toContain('Connection lost'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_lldb_command +// --------------------------------------------------------------------------- +describe('debug_lldb_command', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof lldbHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(lldbSchema).toBeDefined(); + expect('debugSessionId' in lldbSchema).toBe(true); + expect('command' in lldbSchema).toBe(true); + expect('timeoutMs' in lldbSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_lldb_commandLogic({ command: 'bt' }, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should run command successfully', async () => { + const { ctx, session } = await createSessionAndContext({ + runCommand: async () => ' frame #0: main\n', + }); + + const result = await debug_lldb_commandLogic( + { debugSessionId: session.id, command: 'bt' }, + ctx, + ); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toBe('frame #0: main'); + }); + + it('should pass timeoutMs through to runCommand', async () => { + let receivedOpts: { timeoutMs?: number } | undefined; + const { ctx, session } = await createSessionAndContext({ + runCommand: async (_cmd: string, opts?: { timeoutMs?: number }) => { + receivedOpts = opts; + return 'ok'; + }, + }); + + await debug_lldb_commandLogic( + { debugSessionId: session.id, command: 'expr x', timeoutMs: 5000 }, + ctx, + ); + + expect(receivedOpts?.timeoutMs).toBe(5000); + }); + + it('should return error when runCommand throws', async () => { + const { ctx, session } = await createSessionAndContext({ + runCommand: async () => { + throw new Error('Command timed out'); + }, + }); + + const result = await debug_lldb_commandLogic( + { debugSessionId: session.id, command: 'expr longRunning()' }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to run LLDB command'); + expect(text).toContain('Command timed out'); + }); + + it('should use current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext({ + runCommand: async () => 'result', + }); + + const result = await debug_lldb_commandLogic({ command: 'po self' }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toBe('result'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_stack +// --------------------------------------------------------------------------- +describe('debug_stack', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof stackHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(stackSchema).toBeDefined(); + expect('debugSessionId' in stackSchema).toBe(true); + expect('threadIndex' in stackSchema).toBe(true); + expect('maxFrames' in stackSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_stackLogic({}, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should return stack output successfully', async () => { + const stackOutput = ' frame #0: 0x0000 main at main.swift:10\n frame #1: 0x0001 start\n'; + const { ctx, session } = await createSessionAndContext({ + getStack: async () => stackOutput, + }); + + const result = await debug_stackLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toBe(stackOutput.trim()); + }); + + it('should pass threadIndex and maxFrames through', async () => { + let receivedOpts: { threadIndex?: number; maxFrames?: number } | undefined; + const { ctx, session } = await createSessionAndContext({ + getStack: async (opts?: { threadIndex?: number; maxFrames?: number }) => { + receivedOpts = opts; + return 'frame #0'; + }, + }); + + await debug_stackLogic({ debugSessionId: session.id, threadIndex: 2, maxFrames: 5 }, ctx); + + expect(receivedOpts?.threadIndex).toBe(2); + expect(receivedOpts?.maxFrames).toBe(5); + }); + + it('should return error when getStack throws', async () => { + const { ctx, session } = await createSessionAndContext({ + getStack: async () => { + throw new Error('Process not stopped'); + }, + }); + + const result = await debug_stackLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to get stack'); + expect(text).toContain('Process not stopped'); + }); + + it('should use current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext({ + getStack: async () => 'frame #0: main', + }); + + const result = await debug_stackLogic({}, ctx); + + expect(result.isError).toBe(false); + expect(result.content[0].text).toBe('frame #0: main'); + }); + }); +}); + +// --------------------------------------------------------------------------- +// debug_variables +// --------------------------------------------------------------------------- +describe('debug_variables', () => { + beforeEach(() => { + sessionStore.clear(); + }); + + describe('Export Field Validation', () => { + it('should have handler function', () => { + expect(typeof variablesHandler).toBe('function'); + }); + + it('should expose schema with expected keys', () => { + expect(variablesSchema).toBeDefined(); + expect('debugSessionId' in variablesSchema).toBe(true); + expect('frameIndex' in variablesSchema).toBe(true); + }); + }); + + describe('Handler Requirements', () => { + it('should handle missing debug session gracefully', async () => { + const ctx = createTestContext(); + + const result = await debug_variablesLogic({}, ctx); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('No active debug session'); + }); + }); + + describe('Logic Behavior', () => { + it('should return variables output successfully', async () => { + const variablesOutput = ' (Int) x = 42\n (String) name = "hello"\n'; + const { ctx, session } = await createSessionAndContext({ + getVariables: async () => variablesOutput, + }); + + const result = await debug_variablesLogic({ debugSessionId: session.id }, ctx); + + expect(result.isError).toBe(false); + const text = result.content[0].text; + expect(text).toBe(variablesOutput.trim()); + }); + + it('should pass frameIndex through', async () => { + let receivedOpts: { frameIndex?: number } | undefined; + const { ctx, session } = await createSessionAndContext({ + getVariables: async (opts?: { frameIndex?: number }) => { + receivedOpts = opts; + return 'x = 1'; + }, + }); + + await debug_variablesLogic({ debugSessionId: session.id, frameIndex: 3 }, ctx); + + expect(receivedOpts?.frameIndex).toBe(3); + }); + + it('should return error when getVariables throws', async () => { + const { ctx, session } = await createSessionAndContext({ + getVariables: async () => { + throw new Error('Frame index out of range'); + }, + }); + + const result = await debug_variablesLogic( + { debugSessionId: session.id, frameIndex: 999 }, + ctx, + ); + + expect(result.isError).toBe(true); + const text = result.content[0].text; + expect(text).toContain('Failed to get variables'); + expect(text).toContain('Frame index out of range'); + }); + + it('should use current session when debugSessionId is omitted', async () => { + const { ctx } = await createSessionAndContext({ + getVariables: async () => 'y = 99', + }); + + const result = await debug_variablesLogic({}, ctx); + + expect(result.isError).toBe(false); + expect(result.content[0].text).toBe('y = 99'); + }); + }); +}); diff --git a/src/server/server-state.ts b/src/server/server-state.ts index 0b32d62e..573a8c13 100644 --- a/src/server/server-state.ts +++ b/src/server/server-state.ts @@ -10,4 +10,8 @@ export function setServer(server: McpServer): void { serverInstance = server; } +export function __resetServerStateForTests(): void { + serverInstance = undefined; +} + export { serverInstance as server }; diff --git a/src/smoke-tests/__tests__/cli-surface.test.ts b/src/smoke-tests/__tests__/cli-surface.test.ts new file mode 100644 index 00000000..da170cf5 --- /dev/null +++ b/src/smoke-tests/__tests__/cli-surface.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect } from 'vitest'; +import { execSync } from 'child_process'; +import { resolve } from 'path'; + +const CLI = resolve(__dirname, '../../../build/cli.js'); +const cliEnv = (() => { + const env = { ...process.env, NO_COLOR: '1' }; + // Remove test environment markers so the CLI binary runs in production mode + delete env.VITEST; + delete env.NODE_ENV; + return env; +})(); +const run = (args: string): string => { + return execSync(`node ${CLI} ${args}`, { + encoding: 'utf8', + timeout: 15_000, + env: cliEnv, + }); +}; + +const runMayFail = (args: string): { stdout: string; status: number } => { + try { + const stdout = run(args); + return { stdout, status: 0 }; + } catch (err: unknown) { + const error = err as { stdout?: string; stderr?: string; status?: number }; + return { + stdout: (error.stdout ?? '') + (error.stderr ?? ''), + status: error.status ?? 1, + }; + } +}; + +describe('CLI Surface (e2e)', () => { + describe('top-level', () => { + it('--help shows usage info', () => { + const output = run('--help'); + expect(output).toContain('Usage:'); + expect(output).toContain('xcodebuildmcp'); + expect(output).toContain('Commands:'); + }); + + it('--version prints a semver string', () => { + const output = run('--version').trim(); + expect(output).toMatch(/^\d+\.\d+\.\d+/); + }); + + it('tools command lists available tools', () => { + const output = run('tools'); + expect(output).toContain('Available tools'); + expect(output).toContain('simulator:'); + expect(output).toContain('build'); + }); + }); + + describe('workflow subcommands', () => { + const workflows = [ + 'simulator', + 'simulator-management', + 'device', + 'macos', + 'project-discovery', + 'project-scaffolding', + 'swift-package', + 'logging', + 'debugging', + 'ui-automation', + 'utilities', + ]; + + it.each(workflows)('%s --help shows workflow help', (workflow) => { + const output = run(`${workflow} --help`); + expect(output).toContain('Commands:'); + }); + }); + + describe('tool-specific help', () => { + const toolCases = [ + { workflow: 'simulator', tool: 'build', expected: '--scheme' }, + { workflow: 'simulator', tool: 'list-sims', expected: '--help' }, + { workflow: 'device', tool: 'build', expected: '--scheme' }, + { workflow: 'swift-package', tool: 'build', expected: '--package-path' }, + { workflow: 'project-discovery', tool: 'list-schemes', expected: '--project-path' }, + { workflow: 'ui-automation', tool: 'tap', expected: '--simulator-id' }, + { workflow: 'utilities', tool: 'clean', expected: '--scheme' }, + ]; + + it.each(toolCases)( + '$workflow $tool --help shows parameter docs', + ({ workflow, tool, expected }) => { + const output = run(`${workflow} ${tool} --help`); + expect(output).toContain(expected); + }, + ); + }); + + describe('tool invocation', () => { + it('invalid tool returns error', () => { + const result = runMayFail('simulator nonexistent-tool'); + expect(result.status).not.toBe(0); + }); + + it('tool with --output json returns valid JSON', () => { + // list_sims is a good candidate -- it will fail to run xcrun but should + // return structured JSON output even on error + const result = runMayFail('simulator list-sims --output json'); + // Even if the tool fails (no xcrun), the output format should be JSON + if (result.status === 0) { + const parsed = JSON.parse(result.stdout); + expect(parsed).toBeDefined(); + } + // If it fails, that's also acceptable on non-macOS platforms + }); + + it('missing required args produces user-friendly error', () => { + // build requires --scheme + const result = runMayFail('simulator build'); + const output = result.stdout.toLowerCase(); + // Should mention the missing requirement + expect( + output.includes('required') || + output.includes('scheme') || + output.includes('error') || + output.includes('must provide') || + output.includes('missing'), + ).toBe(true); + }); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts b/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts new file mode 100644 index 00000000..1a5fab02 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts @@ -0,0 +1,455 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + xcodebuild: { success: true, output: 'Build Succeeded' }, + devicectl: { success: true, output: '{}' }, + 'xctrace list devices': { success: true, output: 'No devices found.' }, + open: { success: true, output: '' }, + kill: { success: true, output: '' }, + pkill: { success: true, output: '' }, + 'defaults read': { success: true, output: 'com.example.MyApp' }, + PlistBuddy: { success: true, output: 'com.example.MyApp' }, + xcresulttool: { success: true, output: '{}' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Device and macOS Tool Invocation (e2e)', () => { + describe('device tools', () => { + it('build_device captures xcodebuild command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_device', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('MyApp'))).toBe(true); + }); + + it('test_device captures xcodebuild test command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + deviceId: 'AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'test_device', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('test'))).toBe(true); + }); + + it('launch_app_device captures devicectl launch command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + deviceId: 'AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE', + bundleId: 'com.example.MyApp', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'launch_app_device', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('devicectl') && c.includes('launch'))).toBe(true); + }); + + it('stop_app_device captures devicectl terminate command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + deviceId: 'AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'stop_app_device', + arguments: { processId: 12345 }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('devicectl') && c.includes('terminate'))).toBe( + true, + ); + }); + + it('install_app_device captures devicectl install command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + deviceId: 'AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'install_app_device', + arguments: { appPath: '/path/to/MyApp.app' }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('devicectl') && c.includes('install'))).toBe(true); + }); + + it('get_device_app_path captures xcodebuild showBuildSettings command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'get_device_app_path', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some((c) => c.includes('xcodebuild') && c.includes('-showBuildSettings')), + ).toBe(true); + }); + + it('list_devices captures devicectl or xctrace command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'list_devices', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + expect(harness.capturedCommands.length).toBeGreaterThan(0); + }); + }); + + describe('project discovery tools', () => { + it('discover_projs responds with content', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'discover_projs', + arguments: { workspaceRoot: '/path/to/workspace' }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('get_app_bundle_id responds with content', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'get_app_bundle_id', + arguments: { appPath: '/path/to/MyApp.app' }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('get_mac_bundle_id responds with content', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'get_mac_bundle_id', + arguments: { appPath: '/path/to/MyApp.app' }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('macOS tools', () => { + it('build_macos captures xcodebuild command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyMacApp', + projectPath: '/path/to/MyMacApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_macos', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('MyMacApp'))).toBe( + true, + ); + }); + + it('build_run_macos captures xcodebuild and open commands', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyMacApp', + projectPath: '/path/to/MyMacApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_run_macos', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild'))).toBe(true); + }); + + it('test_macos captures xcodebuild test command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyMacApp', + projectPath: '/path/to/MyMacApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'test_macos', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('test'))).toBe(true); + }); + + it('launch_mac_app responds with content', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'launch_mac_app', + arguments: { appPath: '/path/to/MyMacApp.app' }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('stop_mac_app captures kill command with processId', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'stop_mac_app', + arguments: { processId: 54321 }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('kill') && c.includes('54321'))).toBe(true); + }); + + it('stop_mac_app captures pkill command with appName', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'stop_mac_app', + arguments: { appName: 'MyMacApp' }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('MyMacApp'))).toBe(true); + }); + + it('get_mac_app_path captures xcodebuild showBuildSettings command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyMacApp', + projectPath: '/path/to/MyMacApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'get_mac_app_path', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some((c) => c.includes('xcodebuild') && c.includes('-showBuildSettings')), + ).toBe(true); + }); + }); + + describe('error handling', () => { + it('build_device returns error when session defaults missing', async () => { + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); + + const result = await harness.client.callTool({ + name: 'build_device', + arguments: {}, + }); + + const content = 'content' in result ? result.content : []; + const isError = 'isError' in result ? result.isError : false; + const hasErrorText = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('required') || + c.text.toLowerCase().includes('must provide') || + c.text.toLowerCase().includes('provide')), + ); + expect(isError || hasErrorText).toBe(true); + }); + + it('build_macos returns error when session defaults missing', async () => { + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); + + const result = await harness.client.callTool({ + name: 'build_macos', + arguments: {}, + }); + + const content = 'content' in result ? result.content : []; + const isError = 'isError' in result ? result.isError : false; + const hasErrorText = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('required') || + c.text.toLowerCase().includes('must provide') || + c.text.toLowerCase().includes('provide')), + ); + expect(isError || hasErrorText).toBe(true); + }); + + it('stop_mac_app returns error when no appName or processId provided', async () => { + const result = await harness.client.callTool({ + name: 'stop_mac_app', + arguments: {}, + }); + + const content = 'content' in result ? result.content : []; + const isError = 'isError' in result ? result.isError : false; + const hasErrorText = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && + typeof c.text === 'string' && + (c.text.toLowerCase().includes('either') || + c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('must be provided')), + ); + expect(isError || hasErrorText).toBe(true); + }); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-discovery.test.ts b/src/smoke-tests/__tests__/e2e-mcp-discovery.test.ts new file mode 100644 index 00000000..2ee7068f --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-discovery.test.ts @@ -0,0 +1,199 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { loadManifest } from '../../core/manifest/load-manifest.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness(); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Discovery (e2e)', () => { + it('responds to listTools', async () => { + const result = await harness.client.listTools(); + expect(result.tools).toBeDefined(); + expect(result.tools.length).toBeGreaterThan(0); + }); + + it('returns the expected number of tools for all-workflows config', async () => { + const result = await harness.client.listTools(); + + // Count expected MCP-visible tools from manifest (static tools only) + const manifest = loadManifest(); + let manifestMcpTools = 0; + for (const tool of manifest.tools.values()) { + if (tool.availability.mcp) { + manifestMcpTools++; + } + } + + // Actual count may exceed manifest count due to dynamic tool registration + // (e.g., xcode-tools bridge) and may be less due to predicate filtering. + // Assert a reasonable lower bound to catch registration regressions. + expect(result.tools.length).toBeGreaterThan(50); + // Every manifest MCP tool should be registered (minus predicate-gated ones) + expect(result.tools.length).toBeGreaterThanOrEqual(manifestMcpTools - 10); + }); + + it('every tool has an inputSchema with type "object"', async () => { + const result = await harness.client.listTools(); + for (const tool of result.tools) { + expect(tool.inputSchema).toBeDefined(); + expect(tool.inputSchema.type).toBe('object'); + } + }); + + it('every tool has a non-empty description', async () => { + const result = await harness.client.listTools(); + for (const tool of result.tools) { + expect(tool.description).toBeTruthy(); + expect(tool.description!.length).toBeGreaterThan(0); + } + }); + + it('every tool has a non-empty name', async () => { + const result = await harness.client.listTools(); + for (const tool of result.tools) { + expect(tool.name).toBeTruthy(); + expect(tool.name.length).toBeGreaterThan(0); + } + }); + + it('includes session management tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('session_set_defaults'); + expect(names).toContain('session_show_defaults'); + expect(names).toContain('session_clear_defaults'); + }); + + it('excludes workflow discovery when experimentalWorkflowDiscovery is disabled', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + // manage-workflows requires experimentalWorkflowDiscovery predicate + // which is disabled by default -- it should NOT appear + expect(names).not.toContain('manage-workflows'); + }); + + it('includes simulator workflow tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('build_sim'); + expect(names).toContain('list_sims'); + expect(names).toContain('boot_sim'); + }); + + it('includes swift package tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('swift_package_build'); + expect(names).toContain('swift_package_test'); + }); + + it('includes device workflow tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('build_device'); + expect(names).toContain('list_devices'); + }); + + it('includes macOS workflow tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('build_macos'); + expect(names).toContain('build_run_macos'); + }); + + it('includes ui-automation tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('tap'); + expect(names).toContain('swipe'); + expect(names).toContain('screenshot'); + expect(names).toContain('snapshot_ui'); + }); + + it('includes project discovery tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('discover_projs'); + expect(names).toContain('list_schemes'); + }); + + it('includes debugging tools when debug is enabled', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('debug_attach_sim'); + expect(names).toContain('debug_breakpoint_add'); + expect(names).toContain('debug_stack'); + }); + + it('includes logging tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('start_sim_log_cap'); + expect(names).toContain('stop_sim_log_cap'); + }); + + it('includes project scaffolding tools', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + expect(names).toContain('scaffold_ios_project'); + expect(names).toContain('scaffold_macos_project'); + }); + + it('tools have annotations where expected', async () => { + const result = await harness.client.listTools(); + + // build_sim should have destructiveHint annotation + const buildSim = result.tools.find((t) => t.name === 'build_sim'); + expect(buildSim).toBeDefined(); + expect(buildSim!.annotations).toBeDefined(); + + // list_sims should have readOnlyHint annotation + const listSims = result.tools.find((t) => t.name === 'list_sims'); + expect(listSims).toBeDefined(); + expect(listSims!.annotations).toBeDefined(); + }); + + it('no duplicate tool names', async () => { + const result = await harness.client.listTools(); + const names = result.tools.map((t) => t.name); + const uniqueNames = new Set(names); + expect(uniqueNames.size).toBe(names.length); + }); + + it('every MCP-available, predicate-free tool in an enabled workflow is registered', async () => { + const result = await harness.client.listTools(); + const registeredNames = new Set(result.tools.map((t) => t.name)); + const manifest = loadManifest(); + + // Collect tool IDs from workflows that are both MCP-available AND predicate-free + // (workflows with predicates may be excluded at runtime) + const toolIdsInEnabledWorkflows = new Set(); + for (const workflow of manifest.workflows.values()) { + if (workflow.availability.mcp && workflow.predicates.length === 0) { + for (const toolId of workflow.tools) { + toolIdsInEnabledWorkflows.add(toolId); + } + } + } + + const missingTools: string[] = []; + for (const [toolId, tool] of manifest.tools) { + if (tool.availability.mcp && tool.predicates.length === 0) { + if (!toolIdsInEnabledWorkflows.has(toolId)) continue; + const mcpName = tool.names.mcp; + if (!registeredNames.has(mcpName)) { + missingTools.push(mcpName); + } + } + } + + expect(missingTools).toEqual([]); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts new file mode 100644 index 00000000..fe2455ee --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + xcrun: { success: true, output: '' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Doctor Tool (e2e)', () => { + it('doctor returns diagnostic content', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'doctor', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const hasText = + Array.isArray(content) && + content.some( + (c) => 'text' in c && typeof c.text === 'string' && c.text.includes('XcodeBuildMCP Doctor'), + ); + expect(hasText).toBe(true); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts b/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts new file mode 100644 index 00000000..221c3bc1 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts @@ -0,0 +1,193 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + 'simctl list devices': { + success: false, + output: 'simctl error: unable to enumerate devices', + }, + xcodebuild: { + success: false, + output: 'xcodebuild: error: The workspace does not exist.', + }, + 'swift build': { + success: false, + output: 'error: build failed', + }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +beforeEach(async () => { + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); +}); + +function extractText(result: unknown): string { + const r = result as { content?: Array<{ text?: string }> }; + if (!r.content || !Array.isArray(r.content)) return ''; + return r.content.map((c) => c.text ?? '').join('\n'); +} + +function isErrorResponse(result: unknown): boolean { + const r = result as { isError?: boolean; content?: Array<{ text?: string }> }; + if (r.isError) return true; + const text = extractText(result).toLowerCase(); + return ( + text.includes('error') || + text.includes('fail') || + text.includes('missing') || + text.includes('required') + ); +} + +describe('MCP Error Paths (e2e)', () => { + describe('missing session defaults', () => { + it('build_sim errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('build_device errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'build_device', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('build_macos errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'build_macos', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('clean errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'clean', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('test_sim errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'test_sim', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('tap errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'tap', + arguments: { x: 100, y: 200 }, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('boot_sim errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'boot_sim', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + + it('show_build_settings errors without session defaults', async () => { + const result = await harness.client.callTool({ + name: 'show_build_settings', + arguments: {}, + }); + expect(isErrorResponse(result)).toBe(true); + }); + }); + + describe('command failure propagation', () => { + it('build_sim propagates xcodebuild failure', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + simulatorId: 'AAAAAAAA-1111-2222-3333-444444444444', + }, + }); + + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + + expect(isErrorResponse(result)).toBe(true); + const text = extractText(result).toLowerCase(); + expect(text).toContain('fail'); + }); + + it('swift_package_build propagates swift build failure', async () => { + const result = await harness.client.callTool({ + name: 'swift_package_build', + arguments: { packagePath: '/path/to/package' }, + }); + + expect(isErrorResponse(result)).toBe(true); + const text = extractText(result).toLowerCase(); + expect(text).toContain('fail'); + }); + + it('list_sims propagates simctl failure', async () => { + const result = await harness.client.callTool({ + name: 'list_sims', + arguments: {}, + }); + + expect(isErrorResponse(result)).toBe(true); + const text = extractText(result).toLowerCase(); + expect(text).toContain('fail'); + }); + }); + + describe('invalid parameter combinations', () => { + it('session_set_defaults resolves both projectPath and workspacePath by keeping workspacePath', async () => { + const result = await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + projectPath: '/path/to/MyApp.xcodeproj', + workspacePath: '/path/to/MyApp.xcworkspace', + }, + }); + + const text = extractText(result); + expect(text).toContain('keeping workspacePath'); + }); + + it('build_sim still errors when only scheme is set without project or simulator', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { scheme: 'MyApp' }, + }); + + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + + expect(isErrorResponse(result)).toBe(true); + }); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts b/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts new file mode 100644 index 00000000..303d7688 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts @@ -0,0 +1,327 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + 'simctl list devices': { + success: true, + output: JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [ + { + name: 'iPhone 16 Pro', + udid: 'AAAAAAAA-1111-2222-3333-444444444444', + state: 'Shutdown', + isAvailable: true, + }, + ], + }, + }), + }, + xcodebuild: { success: true, output: 'Build Succeeded' }, + 'swift build': { success: true, output: 'Build complete!' }, + 'simctl boot': { success: true, output: '' }, + 'xctrace list devices': { success: true, output: 'No devices found.' }, + 'axe tap': { success: true, output: 'Tap performed at (100, 200)' }, + 'simctl io': { success: true, output: '/tmp/screenshot.png' }, + devicectl: { success: true, output: '{}' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Tool Invocation (e2e)', () => { + describe('every tool responds to callTool', () => { + it('all registered tools return a response when called with empty args', async () => { + const { tools } = await harness.client.listTools(); + const results: { name: string; ok: boolean; hasContent: boolean }[] = []; + + for (const tool of tools) { + try { + const result = await harness.client.callTool({ + name: tool.name, + arguments: {}, + }); + + const content = 'content' in result ? result.content : undefined; + results.push({ + name: tool.name, + ok: true, + hasContent: Array.isArray(content) && content.length > 0, + }); + } catch { + // MCP protocol errors are acceptable for tools with required params + results.push({ name: tool.name, ok: true, hasContent: false }); + } + } + + expect(results.length).toBe(tools.length); + for (const r of results) { + expect(r.ok).toBe(true); + } + }, 60_000); + }); + + describe('representative tools with valid args', () => { + it('list_sims captures simctl command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'list_sims', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('simctl') && c.includes('list'))).toBe(true); + }); + + it('build_sim captures xcodebuild command with scheme', async () => { + // Session-aware tools require session defaults to be set first + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + simulatorId: 'AAAAAAAA-1111-2222-3333-444444444444', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('MyApp'))).toBe(true); + }); + + it('clean captures xcodebuild clean command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'clean', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('clean'))).toBe(true); + }); + + it('swift_package_build captures swift build command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'swift_package_build', + arguments: { + packagePath: '/path/to/package', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('swift') && c.includes('build'))).toBe(true); + }); + + it('boot_sim captures simctl boot command', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + simulatorId: 'AAAAAAAA-1111-2222-3333-444444444444', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'boot_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('simctl') && c.includes('boot'))).toBe(true); + }); + + it('list_schemes responds with content', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + projectPath: '/path/to/MyApp.xcodeproj', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'list_schemes', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + expect(harness.capturedCommands.length).toBeGreaterThan(0); + }); + + it('list_devices responds with content', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'list_devices', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + expect(harness.capturedCommands.length).toBeGreaterThan(0); + }); + + it('session_set_defaults works without external commands', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'TestScheme', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('session_show_defaults returns current defaults', async () => { + const result = await harness.client.callTool({ + name: 'session_show_defaults', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('show_build_settings responds with content', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + projectPath: '/path/to/MyApp.xcodeproj', + scheme: 'MyApp', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'show_build_settings', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('error handling', () => { + it('returns error for missing required args', async () => { + // Clear any session defaults from previous tests + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); + + // build_sim requires scheme + projectPath/workspacePath + simulatorId/simulatorName + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + + const content = 'content' in result ? result.content : []; + const isError = 'isError' in result ? result.isError : false; + const hasErrorText = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('required') || + c.text.toLowerCase().includes('must provide') || + c.text.toLowerCase().includes('invalid') || + c.text.toLowerCase().includes('missing') || + c.text.toLowerCase().includes('provide')), + ); + expect(isError || hasErrorText).toBe(true); + }); + + it('returns error response for non-existent tool', async () => { + // The MCP SDK may either throw or return an error response + // depending on the server implementation + let threw = false; + let errorMessage = ''; + try { + const result = await harness.client.callTool({ + name: 'this_tool_does_not_exist', + arguments: {}, + }); + // If it didn't throw, check for error in the response + const isError = 'isError' in result ? result.isError : false; + const content = 'content' in result ? result.content : []; + const hasError = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && typeof c.text === 'string' && c.text.toLowerCase().includes('error'), + ); + expect(isError || hasError).toBe(true); + } catch (err: unknown) { + threw = true; + errorMessage = (err as Error).message; + } + + if (threw) { + expect(errorMessage.toLowerCase()).toMatch(/not found|unknown|error/); + } + }); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts b/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts new file mode 100644 index 00000000..1dc5a6fe --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + 'simctl spawn': { success: true, output: '' }, + 'log collect': { success: true, output: 'Log captured' }, + devicectl: { success: true, output: '{}' }, + xcrun: { success: true, output: '' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Logging Tools (e2e)', () => { + it('start_sim_log_cap requires simulatorId and bundleId via session', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + simulatorId: 'AAAAAAAA-1111-2222-3333-444444444444', + bundleId: 'com.example.TestApp', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'start_sim_log_cap', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('stop_sim_log_cap returns error for unknown session', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'stop_sim_log_cap', + arguments: { + logSessionId: 'nonexistent-session-id', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const isError = 'isError' in result ? result.isError : false; + const hasErrorText = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('not found') || + c.text.toLowerCase().includes('invalid')), + ); + expect(isError || hasErrorText).toBe(true); + }); + + it('start_device_log_cap requires deviceId and bundleId via session', async () => { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + deviceId: 'BBBBBBBB-1111-2222-3333-444444444444', + bundleId: 'com.example.TestApp', + }, + }); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'start_device_log_cap', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('stop_device_log_cap returns error for unknown session', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'stop_device_log_cap', + arguments: { + logSessionId: 'nonexistent-device-session-id', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const isError = 'isError' in result ? result.isError : false; + const hasErrorText = + Array.isArray(content) && + content.some( + (c) => + 'text' in c && + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('not found') || + c.text.toLowerCase().includes('failed')), + ); + expect(isError || hasErrorText).toBe(true); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts new file mode 100644 index 00000000..3d1f4870 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: {}, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Project Scaffolding Tools (e2e)', () => { + it('scaffold_ios_project returns content with valid args', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'scaffold_ios_project', + arguments: { + projectName: 'TestApp', + outputPath: '/tmp/test-scaffold-ios', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('scaffold_macos_project returns content with valid args', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'scaffold_macos_project', + arguments: { + projectName: 'TestMacApp', + outputPath: '/tmp/test-scaffold-macos', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts new file mode 100644 index 00000000..b38c3360 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts @@ -0,0 +1,188 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + xcodebuild: { success: true, output: 'Build Succeeded' }, + 'simctl list devices': { + success: true, + output: JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [ + { + name: 'iPhone 16 Pro', + udid: 'AAAAAAAA-1111-2222-3333-444444444444', + state: 'Shutdown', + isAvailable: true, + }, + ], + }, + }), + }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +beforeEach(async () => { + // Clear defaults before each test + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); +}); + +function extractText(result: unknown): string { + const r = result as { content?: Array<{ text?: string }> }; + if (!r.content || !Array.isArray(r.content)) return ''; + return r.content.map((c) => c.text ?? '').join('\n'); +} + +describe('MCP Session Management (e2e)', () => { + it('session_set_defaults stores scheme', async () => { + const result = await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { scheme: 'MyApp' }, + }); + + expect(result).toBeDefined(); + const text = extractText(result); + expect(text).toContain('scheme'); + }); + + it('session_show_defaults returns the set defaults', async () => { + // Set some defaults + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { scheme: 'TestApp', projectPath: '/path/to/project' }, + }); + + // Show defaults + const result = await harness.client.callTool({ + name: 'session_show_defaults', + arguments: {}, + }); + + const text = extractText(result); + expect(text).toContain('TestApp'); + expect(text).toContain('/path/to/project'); + }); + + it('session_clear_defaults clears all defaults', async () => { + // Set defaults + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { scheme: 'App', projectPath: '/proj' }, + }); + + // Clear all + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); + + // Show should be empty + const result = await harness.client.callTool({ + name: 'session_show_defaults', + arguments: {}, + }); + + const text = extractText(result); + // Should not contain the previously set values + expect(text).not.toContain('App'); + expect(text).not.toContain('/proj'); + }); + + it('session_clear_defaults clears specific keys', async () => { + // Set multiple defaults + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { scheme: 'KeepThis', projectPath: '/clear/this' }, + }); + + // Clear only projectPath + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { keys: ['projectPath'] }, + }); + + // Show defaults + const result = await harness.client.callTool({ + name: 'session_show_defaults', + arguments: {}, + }); + + const text = extractText(result); + expect(text).toContain('KeepThis'); + expect(text).not.toContain('/clear/this'); + }); + + it('session defaults flow into tool invocations', async () => { + // Set session defaults + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'SessionScheme', + projectPath: '/session/project.xcodeproj', + simulatorId: 'AAAAAAAA-1111-2222-3333-444444444444', + }, + }); + + // Invoke build_sim without explicit scheme/project (should use session defaults) + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + // The captured commands should include the session default scheme + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + const buildCommand = commandStrs.find((c) => c.includes('xcodebuild') && c.includes('-scheme')); + expect(buildCommand).toBeDefined(); + expect(buildCommand).toContain('SessionScheme'); + }); + + it('updating session defaults changes subsequent tool behavior', async () => { + // Set initial defaults + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'FirstScheme', + projectPath: '/first/project.xcodeproj', + simulatorId: 'AAAAAAAA-1111-2222-3333-444444444444', + }, + }); + + // Update scheme via session_set_defaults + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'UpdatedScheme', + }, + }); + + // Invoke build_sim - should use the updated scheme + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + const buildCommand = commandStrs.find((c) => c.includes('xcodebuild') && c.includes('-scheme')); + expect(buildCommand).toBeDefined(); + expect(buildCommand).toContain('UpdatedScheme'); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts b/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts new file mode 100644 index 00000000..0c3bc625 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts @@ -0,0 +1,369 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +const SIM_UUID = 'AAAAAAAA-1111-4222-A333-444444444444'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + 'simctl list devices': { + success: true, + output: JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [ + { + name: 'iPhone 16 Pro', + udid: SIM_UUID, + state: 'Booted', + isAvailable: true, + }, + ], + }, + }), + }, + 'simctl list': { + success: true, + output: JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [ + { + name: 'iPhone 16 Pro', + udid: SIM_UUID, + state: 'Booted', + isAvailable: true, + }, + ], + }, + }), + }, + xcodebuild: { + success: true, + output: + 'Build Succeeded\n' + + 'CODESIGNING_FOLDER_PATH = /tmp/Build/Products/Debug-iphonesimulator/MyApp.app\n' + + 'BUILT_PRODUCTS_DIR = /tmp/Build/Products/Debug-iphonesimulator\n' + + 'FULL_PRODUCT_NAME = MyApp.app\n', + }, + 'simctl boot': { success: true, output: '' }, + 'simctl io': { success: true, output: '/tmp/screenshot.png' }, + 'simctl install': { success: true, output: '' }, + 'simctl launch': { success: true, output: 'com.test.MyApp: 12345' }, + 'simctl terminate': { success: true, output: '' }, + 'simctl erase': { success: true, output: '' }, + 'simctl shutdown': { success: true, output: '' }, + 'simctl ui': { success: true, output: '' }, + 'simctl location': { success: true, output: '' }, + 'simctl status_bar': { success: true, output: '' }, + 'simctl get_app_container': { success: true, output: '/path/to/MyApp.app' }, + 'simctl recordVideo': { success: true, output: '' }, + PlistBuddy: { success: true, output: 'com.test.MyApp' }, + 'open -a Simulator': { success: true, output: '' }, + sips: { success: true, output: '' }, + 'swift -e': { success: true, output: '400,800' }, + axe: { success: true, output: '' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +function setSimulatorSessionDefaults(): Promise { + return harness.client.callTool({ + name: 'session_set_defaults', + arguments: { + scheme: 'MyApp', + projectPath: '/path/to/MyApp.xcodeproj', + simulatorId: SIM_UUID, + bundleId: 'com.test.MyApp', + }, + }); +} + +describe('MCP Simulator Tool Invocation (e2e)', () => { + describe('simulator workflow tools', () => { + it('build_run_sim captures xcodebuild and simctl commands', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'build_run_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild'))).toBe(true); + }); + + it('test_sim captures xcodebuild test command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'test_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('xcodebuild'))).toBe(true); + }); + + it('launch_app_sim captures simctl launch command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'launch_app_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('simctl') && c.includes('launch'))).toBe(true); + }); + + it('stop_app_sim captures simctl terminate command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'stop_app_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('simctl') && c.includes('terminate'))).toBe(true); + }); + + it('install_app_sim responds with content', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'install_app_sim', + arguments: { + appPath: '/tmp/Build/Products/Debug-iphonesimulator/MyApp.app', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('open_sim captures open command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'open_sim', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('open') && c.includes('Simulator'))).toBe(true); + }); + + it('record_sim_video responds with content', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'record_sim_video', + arguments: { + start: true, + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + }); + + it('get_sim_app_path captures xcodebuild -showBuildSettings command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'get_sim_app_path', + arguments: { + platform: 'iOS Simulator', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some((c) => c.includes('xcodebuild') && c.includes('-showBuildSettings')), + ).toBe(true); + }); + + it('screenshot captures simctl io screenshot command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'screenshot', + arguments: { + returnFormat: 'path', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some( + (c) => c.includes('simctl') && c.includes('io') && c.includes('screenshot'), + ), + ).toBe(true); + }); + }); + + describe('simulator-management workflow tools', () => { + it('erase_sims captures simctl erase command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'erase_sims', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('simctl') && c.includes('erase'))).toBe(true); + }); + + it('set_sim_appearance captures simctl ui command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'set_sim_appearance', + arguments: { + mode: 'dark', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some((c) => c.includes('simctl') && c.includes('ui') && c.includes('dark')), + ).toBe(true); + }); + + it('set_sim_location captures simctl location set command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'set_sim_location', + arguments: { + latitude: 37.7749, + longitude: -122.4194, + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some( + (c) => c.includes('simctl') && c.includes('location') && c.includes('set'), + ), + ).toBe(true); + }); + + it('reset_sim_location captures simctl location clear command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'reset_sim_location', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some( + (c) => c.includes('simctl') && c.includes('location') && c.includes('clear'), + ), + ).toBe(true); + }); + + it('sim_statusbar captures simctl status_bar command', async () => { + await setSimulatorSessionDefaults(); + + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'sim_statusbar', + arguments: { + dataNetwork: '5g', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect( + commandStrs.some( + (c) => c.includes('simctl') && c.includes('status_bar') && c.includes('5g'), + ), + ).toBe(true); + }); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts b/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts new file mode 100644 index 00000000..1855645d --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + 'swift build': { success: true, output: 'Build complete!' }, + 'swift package': { success: true, output: 'Package cleaned' }, + 'swift test': { success: true, output: 'Test Suite passed' }, + 'swift run': { success: true, output: 'Running...' }, + pgrep: { success: false, output: '' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +describe('MCP Swift Package Tools (e2e)', () => { + it('swift_package_clean captures swift package clean command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'swift_package_clean', + arguments: { + packagePath: '/path/to/package', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('swift') && c.includes('clean'))).toBe(true); + }); + + it('swift_package_test captures swift test command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'swift_package_test', + arguments: { + packagePath: '/path/to/package', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('swift') && c.includes('test'))).toBe(true); + }); + + it('swift_package_run captures swift run command', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'swift_package_run', + arguments: { + packagePath: '/path/to/package', + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); + expect(commandStrs.some((c) => c.includes('swift') && c.includes('run'))).toBe(true); + }); + + it('swift_package_stop returns content for unknown PID', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'swift_package_stop', + arguments: { + pid: 99999, + }, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const hasText = + Array.isArray(content) && + content.some((c) => 'text' in c && typeof c.text === 'string' && c.text.length > 0); + expect(hasText).toBe(true); + }); + + it('swift_package_list returns content listing processes', async () => { + harness.capturedCommands.length = 0; + const result = await harness.client.callTool({ + name: 'swift_package_list', + arguments: {}, + }); + + expect(result).toBeDefined(); + const content = 'content' in result ? result.content : []; + expect(Array.isArray(content)).toBe(true); + expect(content.length).toBeGreaterThan(0); + + const hasText = + Array.isArray(content) && + content.some((c) => 'text' in c && typeof c.text === 'string' && c.text.length > 0); + expect(hasText).toBe(true); + }); +}); diff --git a/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts b/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts new file mode 100644 index 00000000..5bc3b555 --- /dev/null +++ b/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts @@ -0,0 +1,420 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; + +const SIM_ID = 'AAAAAAAA-1111-2222-3333-444444444444'; + +let harness: McpTestHarness; + +beforeAll(async () => { + harness = await createMcpTestHarness({ + commandResponses: { + 'axe tap': { success: true, output: 'Tap performed' }, + 'axe swipe': { success: true, output: 'Swipe performed' }, + 'axe button': { success: true, output: 'Button pressed' }, + 'axe gesture': { success: true, output: 'Gesture performed' }, + 'axe key': { success: true, output: 'Key pressed' }, + 'axe key-sequence': { success: true, output: 'Key sequence performed' }, + 'axe touch': { success: true, output: 'Touch performed' }, + 'axe type': { success: true, output: 'Type performed' }, + 'axe describe-ui': { + success: true, + output: JSON.stringify({ type: 'application', children: [] }), + }, + 'simctl io': { success: true, output: '/tmp/screenshot.png' }, + 'simctl list devices': { + success: true, + output: JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-18-0': [ + { + name: 'iPhone 16 Pro', + udid: SIM_ID, + state: 'Booted', + isAvailable: true, + }, + ], + }, + }), + }, + sips: { success: true, output: '' }, + swift: { success: true, output: '393,852' }, + }, + }); +}, 30_000); + +afterAll(async () => { + await harness.cleanup(); +}); + +function extractText(result: unknown): string { + const r = result as { content?: Array<{ text?: string }> }; + if (!r.content || !Array.isArray(r.content)) return ''; + return r.content.map((c) => c.text ?? '').join('\n'); +} + +function isErrorResponse(result: unknown): boolean { + const r = result as { isError?: boolean; content?: Array<{ text?: string }> }; + if (r.isError) return true; + if (!r.content || !Array.isArray(r.content)) return false; + return r.content.some( + (c) => + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('required') || + c.text.toLowerCase().includes('missing') || + c.text.toLowerCase().includes('must provide') || + c.text.toLowerCase().includes('provide')), + ); +} + +function getContent(result: unknown): Array<{ type?: string; text?: string }> { + const r = result as { content?: Array<{ type?: string; text?: string }> }; + return Array.isArray(r.content) ? r.content : []; +} + +async function setSimulatorDefaults(): Promise { + await harness.client.callTool({ + name: 'session_set_defaults', + arguments: { simulatorId: SIM_ID }, + }); +} + +async function clearDefaults(): Promise { + await harness.client.callTool({ + name: 'session_clear_defaults', + arguments: { all: true }, + }); +} + +describe('MCP UI Automation Tools (e2e)', () => { + describe('tap', () => { + it('responds via MCP with coordinate tap', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'tap', + arguments: { x: 100, y: 200 }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with element id tap', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'tap', + arguments: { id: 'myButton' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with element label tap', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'tap', + arguments: { label: 'Submit' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('swipe', () => { + it('responds via MCP with swipe coordinates', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'swipe', + arguments: { x1: 100, y1: 200, x2: 100, y2: 600 }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with optional duration and delta', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'swipe', + arguments: { x1: 50, y1: 100, x2: 50, y2: 500, duration: 0.5, delta: 10 }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('button', () => { + it('responds via MCP with home button press', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'button', + arguments: { buttonType: 'home' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with lock button press', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'button', + arguments: { buttonType: 'lock' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('gesture', () => { + it('responds via MCP with scroll-down gesture', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'gesture', + arguments: { preset: 'scroll-down' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with swipe-from-left-edge gesture', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'gesture', + arguments: { preset: 'swipe-from-left-edge' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('key_press', () => { + it('responds via MCP with key press', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'key_press', + arguments: { keyCode: 40 }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('key_sequence', () => { + it('responds via MCP with key sequence', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'key_sequence', + arguments: { keyCodes: [4, 5, 6, 7] }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('long_press', () => { + it('responds via MCP with long press', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'long_press', + arguments: { x: 150, y: 300, duration: 500 }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('screenshot', () => { + it('responds via MCP with screenshot capture', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'screenshot', + arguments: {}, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with path return format', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'screenshot', + arguments: { returnFormat: 'path' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('snapshot_ui', () => { + it('responds via MCP with UI hierarchy', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'snapshot_ui', + arguments: {}, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('touch', () => { + it('responds via MCP with touch down+up', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'touch', + arguments: { x: 200, y: 400, down: true, up: true }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + + it('responds via MCP with touch down only', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'touch', + arguments: { x: 200, y: 400, down: true }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('type_text', () => { + it('responds via MCP with text typing', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'type_text', + arguments: { text: 'Hello World' }, + }); + + const content = getContent(result); + expect(content.length).toBeGreaterThan(0); + }); + }); + + describe('error paths', () => { + it('returns error when simulatorId session default is missing', async () => { + await clearDefaults(); + + const result = await harness.client.callTool({ + name: 'tap', + arguments: { x: 100, y: 200 }, + }); + + expect(isErrorResponse(result)).toBe(true); + const text = extractText(result); + expect(text.toLowerCase()).toMatch(/simulatorid|required|missing|provide/); + }); + + it('returns error for touch with neither down nor up', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'touch', + arguments: { x: 100, y: 200 }, + }); + + expect(isErrorResponse(result)).toBe(true); + }); + + it('returns error for tap with no target specified', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'tap', + arguments: {}, + }); + + expect(isErrorResponse(result)).toBe(true); + }); + + it('returns error for type_text with empty text', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'type_text', + arguments: { text: '' }, + }); + + expect(isErrorResponse(result)).toBe(true); + }); + + it('returns error for key_sequence with empty keyCodes array', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'key_sequence', + arguments: { keyCodes: [] }, + }); + + expect(isErrorResponse(result)).toBe(true); + }); + + it('returns error for swipe with missing coordinates', async () => { + await setSimulatorDefaults(); + harness.capturedCommands.length = 0; + + const result = await harness.client.callTool({ + name: 'swipe', + arguments: {}, + }); + + expect(isErrorResponse(result)).toBe(true); + }); + }); +}); diff --git a/src/smoke-tests/mcp-test-harness.ts b/src/smoke-tests/mcp-test-harness.ts new file mode 100644 index 00000000..2b3febff --- /dev/null +++ b/src/smoke-tests/mcp-test-harness.ts @@ -0,0 +1,215 @@ +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import type { ChildProcess } from 'node:child_process'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import type { CommandExecutor, CommandResponse } from '../utils/CommandExecutor.ts'; +import { + __setTestCommandExecutorOverride, + __setTestFileSystemExecutorOverride, + __clearTestExecutorOverrides, +} from '../utils/command.ts'; +import { + __resetConfigStoreForTests, + initConfigStore, + type RuntimeConfigOverrides, +} from '../utils/config-store.ts'; +import { __resetServerStateForTests } from '../server/server-state.ts'; +import { __resetToolRegistryForTests } from '../utils/tool-registry.ts'; +import { createMockFileSystemExecutor } from '../test-utils/mock-executors.ts'; +import { createServer } from '../server/server.ts'; +import { bootstrapServer } from '../server/bootstrap.ts'; +import { sessionStore } from '../utils/session-store.ts'; +import { + __setTestDebuggerToolContextOverride, + __clearTestDebuggerToolContextOverride, + DebuggerManager, +} from '../utils/debugger/index.ts'; +import { getPackageRoot } from '../core/manifest/load-manifest.ts'; +import { shutdownXcodeToolsBridge } from '../integrations/xcode-tools-bridge/index.ts'; + +export interface CapturedCommand { + command: string[]; + logPrefix?: string; + useShell?: boolean; + opts?: { env?: Record; cwd?: string }; + detached?: boolean; +} + +export interface McpTestHarness { + client: Client; + capturedCommands: CapturedCommand[]; + cleanup(): Promise; +} + +export interface McpTestHarnessOptions { + enabledWorkflows?: string[]; + commandResponses?: Record; +} + +const defaultCommandResponse: CommandResponse = { + success: true, + output: '', + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + process: { pid: 99999 } as ChildProcess, + exitCode: 0, +}; + +export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promise { + const capturedCommands: CapturedCommand[] = []; + + const capturingExecutor: CommandExecutor = async ( + command, + logPrefix, + useShell, + execOpts, + detached, + ) => { + capturedCommands.push({ command, logPrefix, useShell, opts: execOpts, detached }); + + if (opts?.commandResponses) { + const commandStr = command.join(' '); + for (const [pattern, response] of Object.entries(opts.commandResponses)) { + if (commandStr.includes(pattern)) { + return { + ...defaultCommandResponse, + ...response, + }; + } + } + } + + return defaultCommandResponse; + }; + + // Reset all singletons + __resetConfigStoreForTests(); + __resetServerStateForTests(); + __resetToolRegistryForTests(); + sessionStore.clear(); + + // Set executor overrides on the vitest-resolved source modules + __setTestCommandExecutorOverride(capturingExecutor); + __setTestFileSystemExecutorOverride(createMockFileSystemExecutor()); + + // Also set overrides on the built module instances (used by dynamically imported tool handlers) + const buildRoot = resolve(getPackageRoot(), 'build'); + const builtCommandModule = (await import( + pathToFileURL(resolve(buildRoot, 'utils/command.js')).href + )) as { + __setTestCommandExecutorOverride: typeof __setTestCommandExecutorOverride; + __setTestFileSystemExecutorOverride: typeof __setTestFileSystemExecutorOverride; + __clearTestExecutorOverrides: typeof __clearTestExecutorOverrides; + }; + builtCommandModule.__setTestCommandExecutorOverride(capturingExecutor); + builtCommandModule.__setTestFileSystemExecutorOverride(createMockFileSystemExecutor()); + + // Set debugger tool context override (source module) + __setTestDebuggerToolContextOverride({ + executor: capturingExecutor, + debugger: new DebuggerManager(), + }); + + // Set debugger tool context override (built module) + const builtDebuggerModule = (await import( + pathToFileURL(resolve(buildRoot, 'utils/debugger/tool-context.js')).href + )) as { + __setTestDebuggerToolContextOverride: typeof __setTestDebuggerToolContextOverride; + __clearTestDebuggerToolContextOverride: typeof __clearTestDebuggerToolContextOverride; + }; + builtDebuggerModule.__setTestDebuggerToolContextOverride({ + executor: capturingExecutor, + debugger: new DebuggerManager(), + }); + + // Initialize the built config-store module (separate singleton from source module). + // Tool modules loaded from build/ use this config store for schema resolution + // (e.g. session-aware vs legacy schema selection) and requirement validation. + const builtConfigStoreModule = (await import( + pathToFileURL(resolve(buildRoot, 'utils/config-store.js')).href + )) as { + __resetConfigStoreForTests: typeof __resetConfigStoreForTests; + initConfigStore: typeof initConfigStore; + }; + builtConfigStoreModule.__resetConfigStoreForTests(); + const mockFs = createMockFileSystemExecutor(); + await builtConfigStoreModule.initConfigStore({ + cwd: '/test', + fs: mockFs, + overrides: { + debug: true, + disableXcodeAutoSync: true, + } satisfies RuntimeConfigOverrides, + }); + + // Import the built session-store module (separate singleton from source module). + // Session-aware tool handlers in build/ read/write defaults via this store. + const builtSessionStoreModule = (await import( + pathToFileURL(resolve(buildRoot, 'utils/session-store.js')).href + )) as { + sessionStore: typeof sessionStore; + }; + builtSessionStoreModule.sessionStore.clear(); + + // Create server (uses the real createServer + manifest system) + const server = createServer(); + + // Bootstrap with workflows enabled for maximum coverage. + // xcode-ide is excluded: it connects to the real Xcode tools bridge MCP + // server which triggers system permission prompts and requires Xcode. + const allWorkflows = opts?.enabledWorkflows ?? [ + 'simulator', + 'simulator-management', + 'device', + 'macos', + 'project-discovery', + 'project-scaffolding', + 'session-management', + 'swift-package', + 'logging', + 'debugging', + 'ui-automation', + 'utilities', + 'workflow-discovery', + 'doctor', + ]; + + await bootstrapServer(server, { + enabledWorkflows: allWorkflows, + configOverrides: { + debug: true, + disableXcodeAutoSync: true, + }, + fileSystemExecutor: createMockFileSystemExecutor(), + }); + + // Create InMemoryTransport linked pair + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + + // Connect server to one end + await server.connect(serverTransport); + + // Create and connect client to the other end + const client = new Client({ name: 'e2e-test-client', version: '1.0.0' }); + await client.connect(clientTransport); + + return { + client, + capturedCommands, + async cleanup(): Promise { + await client.close(); + await server.close(); + await shutdownXcodeToolsBridge(); + __clearTestExecutorOverrides(); + builtCommandModule.__clearTestExecutorOverrides(); + __clearTestDebuggerToolContextOverride(); + builtDebuggerModule.__clearTestDebuggerToolContextOverride(); + __resetConfigStoreForTests(); + builtConfigStoreModule.__resetConfigStoreForTests(); + __resetServerStateForTests(); + __resetToolRegistryForTests(); + sessionStore.clear(); + builtSessionStoreModule.sessionStore.clear(); + }, + }; +} diff --git a/src/utils/command.ts b/src/utils/command.ts index d26170b9..9d85fb3d 100644 --- a/src/utils/command.ts +++ b/src/utils/command.ts @@ -221,12 +221,29 @@ const defaultFileSystemExecutor: FileSystemExecutor = { }, }; +let _testCommandExecutorOverride: CommandExecutor | null = null; +let _testFileSystemExecutorOverride: FileSystemExecutor | null = null; + +export function __setTestCommandExecutorOverride(executor: CommandExecutor | null): void { + _testCommandExecutorOverride = executor; +} + +export function __setTestFileSystemExecutorOverride(executor: FileSystemExecutor | null): void { + _testFileSystemExecutorOverride = executor; +} + +export function __clearTestExecutorOverrides(): void { + _testCommandExecutorOverride = null; + _testFileSystemExecutorOverride = null; +} + /** * Get default command executor with test safety * Throws error if used in test environment to ensure proper mocking */ export function getDefaultCommandExecutor(): CommandExecutor { if (process.env.VITEST === 'true' || process.env.NODE_ENV === 'test') { + if (_testCommandExecutorOverride) return _testCommandExecutorOverride; throw new Error( `🚨 REAL SYSTEM EXECUTOR DETECTED IN TEST! 🚨\n` + `This test is trying to use the default command executor instead of a mock.\n` + @@ -244,6 +261,7 @@ export function getDefaultCommandExecutor(): CommandExecutor { */ export function getDefaultFileSystemExecutor(): FileSystemExecutor { if (process.env.VITEST === 'true' || process.env.NODE_ENV === 'test') { + if (_testFileSystemExecutorOverride) return _testFileSystemExecutorOverride; throw new Error( `🚨 REAL FILESYSTEM EXECUTOR DETECTED IN TEST! 🚨\n` + `This test is trying to use the default filesystem executor instead of a mock.\n` + diff --git a/src/utils/debugger/index.ts b/src/utils/debugger/index.ts index 839b9b4e..ea399d82 100644 --- a/src/utils/debugger/index.ts +++ b/src/utils/debugger/index.ts @@ -8,7 +8,11 @@ export function getDefaultDebuggerManager(): DebuggerManager { } export { DebuggerManager } from './debugger-manager.ts'; -export { getDefaultDebuggerToolContext } from './tool-context.ts'; +export { + getDefaultDebuggerToolContext, + __setTestDebuggerToolContextOverride, + __clearTestDebuggerToolContextOverride, +} from './tool-context.ts'; export { resolveSimulatorAppPid } from './simctl.ts'; export { guardUiAutomationAgainstStoppedDebugger } from './ui-automation-guard.ts'; export type { diff --git a/src/utils/debugger/tool-context.ts b/src/utils/debugger/tool-context.ts index 8c880315..c4e5087a 100644 --- a/src/utils/debugger/tool-context.ts +++ b/src/utils/debugger/tool-context.ts @@ -8,7 +8,20 @@ export type DebuggerToolContext = { debugger: DebuggerManager; }; +let _testContextOverride: DebuggerToolContext | null = null; + +export function __setTestDebuggerToolContextOverride(ctx: DebuggerToolContext | null): void { + _testContextOverride = ctx; +} + +export function __clearTestDebuggerToolContextOverride(): void { + _testContextOverride = null; +} + export function getDefaultDebuggerToolContext(): DebuggerToolContext { + if ((process.env.VITEST === 'true' || process.env.NODE_ENV === 'test') && _testContextOverride) { + return _testContextOverride; + } return { executor: getDefaultCommandExecutor(), debugger: getDefaultDebuggerManager(), diff --git a/src/utils/execution/index.ts b/src/utils/execution/index.ts index e7615e0c..4ca96841 100644 --- a/src/utils/execution/index.ts +++ b/src/utils/execution/index.ts @@ -2,7 +2,13 @@ * Focused execution facade. * Prefer importing from 'utils/execution/index.js' instead of the legacy utils barrel. */ -export { getDefaultCommandExecutor, getDefaultFileSystemExecutor } from '../command.ts'; +export { + getDefaultCommandExecutor, + getDefaultFileSystemExecutor, + __setTestCommandExecutorOverride, + __setTestFileSystemExecutorOverride, + __clearTestExecutorOverrides, +} from '../command.ts'; export { getDefaultInteractiveSpawner } from './interactive-process.ts'; // Types diff --git a/src/utils/tool-registry.ts b/src/utils/tool-registry.ts index 156d4763..46024567 100644 --- a/src/utils/tool-registry.ts +++ b/src/utils/tool-registry.ts @@ -168,3 +168,12 @@ export async function updateWorkflowsFromManifest( ): Promise { await registerWorkflowsFromManifest(workflowNames, ctx); } + +export function __resetToolRegistryForTests(): void { + for (const tool of registryState.tools.values()) { + tool.remove(); + } + registryState.tools.clear(); + registryState.enabledWorkflows.clear(); + registryState.currentContext = null; +} diff --git a/vitest.config.ts b/vitest.config.ts index 8be77e6a..87ac5a2e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -20,6 +20,7 @@ export default defineConfig({ '**/experiments/**', '**/__pycache__/**', '**/dist/**', + 'src/smoke-tests/**', ], pool: 'threads', poolOptions: { diff --git a/vitest.smoke.config.ts b/vitest.smoke.config.ts new file mode 100644 index 00000000..6a4e494b --- /dev/null +++ b/vitest.smoke.config.ts @@ -0,0 +1,28 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + globals: true, + include: [ + 'src/smoke-tests/__tests__/**/*.test.ts', + ], + pool: 'forks', + poolOptions: { + forks: { + maxForks: 1, + }, + }, + env: { + NODE_OPTIONS: '--max-old-space-size=4096', + }, + testTimeout: 60000, + hookTimeout: 30000, + teardownTimeout: 10000, + }, + resolve: { + alias: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, + }, +}); From dfad2c93d120e92f051b1d0f90fe9a8e0e489420 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 7 Feb 2026 12:32:55 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(smoke-tests):?= =?UTF-8?q?=20deduplicate=20helpers,=20harden=20harness,=20improve=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract shared extractText, isErrorResponse, getContent into test-helpers.ts - Add resetCapturedCommands() to McpTestHarness interface replacing direct array mutation - Add build-directory guard with actionable error message in createMcpTestHarness - Document dynamic import necessity for built module patching - Replace inline error-checking blocks in device-macos tests with shared isErrorResponse - Narrow cli-surface.test.ts catch type to NodeJS.ErrnoException --- src/smoke-tests/__tests__/cli-surface.test.ts | 6 +- .../__tests__/e2e-mcp-device-macos.test.ts | 79 +++++-------------- .../__tests__/e2e-mcp-doctor.test.ts | 2 +- .../__tests__/e2e-mcp-error-paths.test.ts | 19 +---- .../__tests__/e2e-mcp-invocation.test.ts | 18 ++--- .../__tests__/e2e-mcp-logging.test.ts | 8 +- .../__tests__/e2e-mcp-scaffolding.test.ts | 4 +- .../__tests__/e2e-mcp-sessions.test.ts | 11 +-- .../__tests__/e2e-mcp-simulator.test.ts | 28 +++---- .../__tests__/e2e-mcp-swift-package.test.ts | 10 +-- .../__tests__/e2e-mcp-ui-automation.test.ts | 73 ++++++----------- src/smoke-tests/mcp-test-harness.ts | 12 +++ src/smoke-tests/test-helpers.ts | 26 ++++++ 13 files changed, 127 insertions(+), 169 deletions(-) create mode 100644 src/smoke-tests/test-helpers.ts diff --git a/src/smoke-tests/__tests__/cli-surface.test.ts b/src/smoke-tests/__tests__/cli-surface.test.ts index da170cf5..92896bc4 100644 --- a/src/smoke-tests/__tests__/cli-surface.test.ts +++ b/src/smoke-tests/__tests__/cli-surface.test.ts @@ -23,7 +23,11 @@ const runMayFail = (args: string): { stdout: string; status: number } => { const stdout = run(args); return { stdout, status: 0 }; } catch (err: unknown) { - const error = err as { stdout?: string; stderr?: string; status?: number }; + const error = err as NodeJS.ErrnoException & { + stdout?: string; + stderr?: string; + status?: number; + }; return { stdout: (error.stdout ?? '') + (error.stderr ?? ''), status: error.status ?? 1, diff --git a/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts b/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts index 1a5fab02..cda5297d 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { isErrorResponse } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -34,7 +35,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_device', arguments: {}, @@ -59,7 +60,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'test_device', arguments: {}, @@ -83,7 +84,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'launch_app_device', arguments: {}, @@ -106,7 +107,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'stop_app_device', arguments: { processId: 12345 }, @@ -131,7 +132,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'install_app_device', arguments: { appPath: '/path/to/MyApp.app' }, @@ -155,7 +156,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'get_device_app_path', arguments: {}, @@ -173,7 +174,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }); it('list_devices captures devicectl or xctrace command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'list_devices', arguments: {}, @@ -190,7 +191,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { describe('project discovery tools', () => { it('discover_projs responds with content', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'discover_projs', arguments: { workspaceRoot: '/path/to/workspace' }, @@ -203,7 +204,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }); it('get_app_bundle_id responds with content', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'get_app_bundle_id', arguments: { appPath: '/path/to/MyApp.app' }, @@ -216,7 +217,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }); it('get_mac_bundle_id responds with content', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'get_mac_bundle_id', arguments: { appPath: '/path/to/MyApp.app' }, @@ -239,7 +240,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_macos', arguments: {}, @@ -265,7 +266,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_run_macos', arguments: {}, @@ -289,7 +290,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'test_macos', arguments: {}, @@ -305,7 +306,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }); it('launch_mac_app responds with content', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'launch_mac_app', arguments: { appPath: '/path/to/MyMacApp.app' }, @@ -318,7 +319,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }); it('stop_mac_app captures kill command with processId', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'stop_mac_app', arguments: { processId: 54321 }, @@ -334,7 +335,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }); it('stop_mac_app captures pkill command with appName', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'stop_mac_app', arguments: { appName: 'MyMacApp' }, @@ -358,7 +359,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'get_mac_app_path', arguments: {}, @@ -388,20 +389,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - const content = 'content' in result ? result.content : []; - const isError = 'isError' in result ? result.isError : false; - const hasErrorText = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && - typeof c.text === 'string' && - (c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('required') || - c.text.toLowerCase().includes('must provide') || - c.text.toLowerCase().includes('provide')), - ); - expect(isError || hasErrorText).toBe(true); + expect(isErrorResponse(result)).toBe(true); }); it('build_macos returns error when session defaults missing', async () => { @@ -415,20 +403,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - const content = 'content' in result ? result.content : []; - const isError = 'isError' in result ? result.isError : false; - const hasErrorText = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && - typeof c.text === 'string' && - (c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('required') || - c.text.toLowerCase().includes('must provide') || - c.text.toLowerCase().includes('provide')), - ); - expect(isError || hasErrorText).toBe(true); + expect(isErrorResponse(result)).toBe(true); }); it('stop_mac_app returns error when no appName or processId provided', async () => { @@ -437,19 +412,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - const content = 'content' in result ? result.content : []; - const isError = 'isError' in result ? result.isError : false; - const hasErrorText = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && - typeof c.text === 'string' && - (c.text.toLowerCase().includes('either') || - c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('must be provided')), - ); - expect(isError || hasErrorText).toBe(true); + expect(isErrorResponse(result)).toBe(true); }); }); }); diff --git a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts index fe2455ee..c0d9680a 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts @@ -17,7 +17,7 @@ afterAll(async () => { describe('MCP Doctor Tool (e2e)', () => { it('doctor returns diagnostic content', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'doctor', arguments: {}, diff --git a/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts b/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts index 221c3bc1..ce808c92 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-error-paths.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { extractText, isErrorResponse } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -33,24 +34,6 @@ beforeEach(async () => { }); }); -function extractText(result: unknown): string { - const r = result as { content?: Array<{ text?: string }> }; - if (!r.content || !Array.isArray(r.content)) return ''; - return r.content.map((c) => c.text ?? '').join('\n'); -} - -function isErrorResponse(result: unknown): boolean { - const r = result as { isError?: boolean; content?: Array<{ text?: string }> }; - if (r.isError) return true; - const text = extractText(result).toLowerCase(); - return ( - text.includes('error') || - text.includes('fail') || - text.includes('missing') || - text.includes('required') - ); -} - describe('MCP Error Paths (e2e)', () => { describe('missing session defaults', () => { it('build_sim errors without session defaults', async () => { diff --git a/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts b/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts index 303d7688..f7ee4343 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts @@ -70,7 +70,7 @@ describe('MCP Tool Invocation (e2e)', () => { describe('representative tools with valid args', () => { it('list_sims captures simctl command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'list_sims', arguments: {}, @@ -96,7 +96,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_sim', arguments: {}, @@ -120,7 +120,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'clean', arguments: {}, @@ -136,7 +136,7 @@ describe('MCP Tool Invocation (e2e)', () => { }); it('swift_package_build captures swift build command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swift_package_build', arguments: { @@ -161,7 +161,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'boot_sim', arguments: {}, @@ -184,7 +184,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'list_schemes', arguments: {}, @@ -199,7 +199,7 @@ describe('MCP Tool Invocation (e2e)', () => { }); it('list_devices responds with content', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'list_devices', arguments: {}, @@ -214,7 +214,7 @@ describe('MCP Tool Invocation (e2e)', () => { }); it('session_set_defaults works without external commands', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'session_set_defaults', arguments: { @@ -249,7 +249,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'show_build_settings', arguments: {}, diff --git a/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts b/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts index 1dc5a6fe..3df55c01 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts @@ -28,7 +28,7 @@ describe('MCP Logging Tools (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'start_sim_log_cap', arguments: {}, @@ -41,7 +41,7 @@ describe('MCP Logging Tools (e2e)', () => { }); it('stop_sim_log_cap returns error for unknown session', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'stop_sim_log_cap', arguments: { @@ -77,7 +77,7 @@ describe('MCP Logging Tools (e2e)', () => { }, }); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'start_device_log_cap', arguments: {}, @@ -90,7 +90,7 @@ describe('MCP Logging Tools (e2e)', () => { }); it('stop_device_log_cap returns error for unknown session', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'stop_device_log_cap', arguments: { diff --git a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts index 3d1f4870..811d868f 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts @@ -15,7 +15,7 @@ afterAll(async () => { describe('MCP Project Scaffolding Tools (e2e)', () => { it('scaffold_ios_project returns content with valid args', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'scaffold_ios_project', arguments: { @@ -31,7 +31,7 @@ describe('MCP Project Scaffolding Tools (e2e)', () => { }); it('scaffold_macos_project returns content with valid args', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'scaffold_macos_project', arguments: { diff --git a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts index b38c3360..43bfb334 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { extractText } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -38,12 +39,6 @@ beforeEach(async () => { }); }); -function extractText(result: unknown): string { - const r = result as { content?: Array<{ text?: string }> }; - if (!r.content || !Array.isArray(r.content)) return ''; - return r.content.map((c) => c.text ?? '').join('\n'); -} - describe('MCP Session Management (e2e)', () => { it('session_set_defaults stores scheme', async () => { const result = await harness.client.callTool({ @@ -135,7 +130,7 @@ describe('MCP Session Management (e2e)', () => { }); // Invoke build_sim without explicit scheme/project (should use session defaults) - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_sim', arguments: {}, @@ -173,7 +168,7 @@ describe('MCP Session Management (e2e)', () => { }); // Invoke build_sim - should use the updated scheme - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_sim', arguments: {}, diff --git a/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts b/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts index 0c3bc625..bf5e335b 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts @@ -88,7 +88,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('build_run_sim captures xcodebuild and simctl commands', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'build_run_sim', arguments: {}, @@ -106,7 +106,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('test_sim captures xcodebuild test command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'test_sim', arguments: {}, @@ -124,7 +124,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('launch_app_sim captures simctl launch command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'launch_app_sim', arguments: {}, @@ -142,7 +142,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('stop_app_sim captures simctl terminate command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'stop_app_sim', arguments: {}, @@ -160,7 +160,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('install_app_sim responds with content', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'install_app_sim', arguments: { @@ -175,7 +175,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }); it('open_sim captures open command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'open_sim', arguments: {}, @@ -193,7 +193,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('record_sim_video responds with content', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'record_sim_video', arguments: { @@ -210,7 +210,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('get_sim_app_path captures xcodebuild -showBuildSettings command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'get_sim_app_path', arguments: { @@ -232,7 +232,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('screenshot captures simctl io screenshot command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'screenshot', arguments: { @@ -258,7 +258,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('erase_sims captures simctl erase command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'erase_sims', arguments: {}, @@ -276,7 +276,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('set_sim_appearance captures simctl ui command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'set_sim_appearance', arguments: { @@ -298,7 +298,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('set_sim_location captures simctl location set command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'set_sim_location', arguments: { @@ -323,7 +323,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('reset_sim_location captures simctl location clear command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'reset_sim_location', arguments: {}, @@ -345,7 +345,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { it('sim_statusbar captures simctl status_bar command', async () => { await setSimulatorSessionDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'sim_statusbar', arguments: { diff --git a/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts b/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts index 1855645d..23479654 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts @@ -21,7 +21,7 @@ afterAll(async () => { describe('MCP Swift Package Tools (e2e)', () => { it('swift_package_clean captures swift package clean command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swift_package_clean', arguments: { @@ -39,7 +39,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }); it('swift_package_test captures swift test command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swift_package_test', arguments: { @@ -57,7 +57,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }); it('swift_package_run captures swift run command', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swift_package_run', arguments: { @@ -75,7 +75,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }); it('swift_package_stop returns content for unknown PID', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swift_package_stop', arguments: { @@ -95,7 +95,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }); it('swift_package_list returns content listing processes', async () => { - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swift_package_list', arguments: {}, diff --git a/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts b/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts index 5bc3b555..cbf4a3c2 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-ui-automation.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { extractText, isErrorResponse, getContent } from '../test-helpers.ts'; const SIM_ID = 'AAAAAAAA-1111-2222-3333-444444444444'; @@ -46,32 +47,6 @@ afterAll(async () => { await harness.cleanup(); }); -function extractText(result: unknown): string { - const r = result as { content?: Array<{ text?: string }> }; - if (!r.content || !Array.isArray(r.content)) return ''; - return r.content.map((c) => c.text ?? '').join('\n'); -} - -function isErrorResponse(result: unknown): boolean { - const r = result as { isError?: boolean; content?: Array<{ text?: string }> }; - if (r.isError) return true; - if (!r.content || !Array.isArray(r.content)) return false; - return r.content.some( - (c) => - typeof c.text === 'string' && - (c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('required') || - c.text.toLowerCase().includes('missing') || - c.text.toLowerCase().includes('must provide') || - c.text.toLowerCase().includes('provide')), - ); -} - -function getContent(result: unknown): Array<{ type?: string; text?: string }> { - const r = result as { content?: Array<{ type?: string; text?: string }> }; - return Array.isArray(r.content) ? r.content : []; -} - async function setSimulatorDefaults(): Promise { await harness.client.callTool({ name: 'session_set_defaults', @@ -90,7 +65,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('tap', () => { it('responds via MCP with coordinate tap', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'tap', @@ -103,7 +78,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with element id tap', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'tap', @@ -116,7 +91,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with element label tap', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'tap', @@ -131,7 +106,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('swipe', () => { it('responds via MCP with swipe coordinates', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swipe', @@ -144,7 +119,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with optional duration and delta', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swipe', @@ -159,7 +134,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('button', () => { it('responds via MCP with home button press', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'button', @@ -172,7 +147,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with lock button press', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'button', @@ -187,7 +162,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('gesture', () => { it('responds via MCP with scroll-down gesture', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'gesture', @@ -200,7 +175,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with swipe-from-left-edge gesture', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'gesture', @@ -215,7 +190,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('key_press', () => { it('responds via MCP with key press', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'key_press', @@ -230,7 +205,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('key_sequence', () => { it('responds via MCP with key sequence', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'key_sequence', @@ -245,7 +220,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('long_press', () => { it('responds via MCP with long press', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'long_press', @@ -260,7 +235,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('screenshot', () => { it('responds via MCP with screenshot capture', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'screenshot', @@ -273,7 +248,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with path return format', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'screenshot', @@ -288,7 +263,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('snapshot_ui', () => { it('responds via MCP with UI hierarchy', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'snapshot_ui', @@ -303,7 +278,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('touch', () => { it('responds via MCP with touch down+up', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'touch', @@ -316,7 +291,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('responds via MCP with touch down only', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'touch', @@ -331,7 +306,7 @@ describe('MCP UI Automation Tools (e2e)', () => { describe('type_text', () => { it('responds via MCP with text typing', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'type_text', @@ -359,7 +334,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('returns error for touch with neither down nor up', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'touch', @@ -371,7 +346,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('returns error for tap with no target specified', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'tap', @@ -383,7 +358,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('returns error for type_text with empty text', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'type_text', @@ -395,7 +370,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('returns error for key_sequence with empty keyCodes array', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'key_sequence', @@ -407,7 +382,7 @@ describe('MCP UI Automation Tools (e2e)', () => { it('returns error for swipe with missing coordinates', async () => { await setSimulatorDefaults(); - harness.capturedCommands.length = 0; + harness.resetCapturedCommands(); const result = await harness.client.callTool({ name: 'swipe', diff --git a/src/smoke-tests/mcp-test-harness.ts b/src/smoke-tests/mcp-test-harness.ts index 2b3febff..0251c986 100644 --- a/src/smoke-tests/mcp-test-harness.ts +++ b/src/smoke-tests/mcp-test-harness.ts @@ -1,3 +1,4 @@ +import { existsSync } from 'node:fs'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import type { ChildProcess } from 'node:child_process'; @@ -39,6 +40,7 @@ export interface CapturedCommand { export interface McpTestHarness { client: Client; capturedCommands: CapturedCommand[]; + resetCapturedCommands(): void; cleanup(): Promise; } @@ -94,6 +96,13 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis // Also set overrides on the built module instances (used by dynamically imported tool handlers) const buildRoot = resolve(getPackageRoot(), 'build'); + if (!existsSync(buildRoot)) { + throw new Error( + `Build directory not found at ${buildRoot}. Run "npm run build" before running smoke tests.`, + ); + } + + // Dynamic imports required: built modules are separate JS instances that must be patched independently of vitest-resolved source modules. const builtCommandModule = (await import( pathToFileURL(resolve(buildRoot, 'utils/command.js')).href )) as { @@ -196,6 +205,9 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis return { client, capturedCommands, + resetCapturedCommands(): void { + capturedCommands.length = 0; + }, async cleanup(): Promise { await client.close(); await server.close(); diff --git a/src/smoke-tests/test-helpers.ts b/src/smoke-tests/test-helpers.ts new file mode 100644 index 00000000..0371f2e7 --- /dev/null +++ b/src/smoke-tests/test-helpers.ts @@ -0,0 +1,26 @@ +export function extractText(result: unknown): string { + const r = result as { content?: Array<{ text?: string }> }; + if (!r.content || !Array.isArray(r.content)) return ''; + return r.content.map((c) => c.text ?? '').join('\n'); +} + +export function isErrorResponse(result: unknown): boolean { + const r = result as { isError?: boolean; content?: Array<{ text?: string }> }; + if (r.isError) return true; + if (!r.content || !Array.isArray(r.content)) return false; + return r.content.some( + (c) => + typeof c.text === 'string' && + (c.text.toLowerCase().includes('error') || + c.text.toLowerCase().includes('required') || + c.text.toLowerCase().includes('missing') || + c.text.toLowerCase().includes('must provide') || + c.text.toLowerCase().includes('provide') || + c.text.toLowerCase().includes('fail')), + ); +} + +export function getContent(result: unknown): Array<{ type?: string; text?: string }> { + const r = result as { content?: Array<{ type?: string; text?: string }> }; + return Array.isArray(r.content) ? r.content : []; +} From 8600c243f6987b2cb9faa6ec9e91637787c6514a Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 7 Feb 2026 12:57:52 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=90=9B=20fix(smoke-tests):=20address?= =?UTF-8?q?=20code=20review=20findings=20for=20test=20reliability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove false-positive "provide" match from isErrorResponse (keep "must provide") - Set exitCode dynamically based on success in command response mock - Sort commandResponses by pattern length to prevent shorter patterns shadowing longer ones - Consolidate createMockFileSystemExecutor() calls to single shared instance - Add expectContent() helper and replace ~40 inline content-extraction patterns - Replace remaining inline error-checking blocks with shared isErrorResponse - Guard tool.remove() with try-catch in __resetToolRegistryForTests for safe cleanup - Build before smoke tests in test:smoke npm script to prevent stale-build issues --- package.json | 2 +- .../__tests__/e2e-mcp-device-macos.test.ts | 87 ++++--------------- .../__tests__/e2e-mcp-doctor.test.ts | 13 ++- .../__tests__/e2e-mcp-invocation.test.ts | 79 +++-------------- .../__tests__/e2e-mcp-logging.test.ts | 49 ++--------- .../__tests__/e2e-mcp-scaffolding.test.ts | 11 +-- .../__tests__/e2e-mcp-sessions.test.ts | 7 +- .../__tests__/e2e-mcp-simulator.test.ts | 71 ++++----------- .../__tests__/e2e-mcp-swift-package.test.ts | 38 ++------ src/smoke-tests/mcp-test-harness.ts | 13 +-- src/smoke-tests/test-helpers.ts | 9 +- src/utils/tool-registry.ts | 6 +- 12 files changed, 93 insertions(+), 292 deletions(-) diff --git a/package.json b/package.json index 93bdd6d7..17612f89 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "docs:update": "npx tsx scripts/update-tools-docs.ts", "docs:update:dry-run": "npx tsx scripts/update-tools-docs.ts --dry-run --verbose", "test": "vitest run", - "test:smoke": "vitest run --config vitest.smoke.config.ts", + "test:smoke": "npm run build && vitest run --config vitest.smoke.config.ts", "test:watch": "vitest", "test:ui": "vitest --ui", "test:coverage": "vitest run --coverage" diff --git a/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts b/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts index cda5297d..b6ddc2f3 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-device-macos.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; -import { isErrorResponse } from '../test-helpers.ts'; +import { isErrorResponse, expectContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -41,10 +41,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('MyApp'))).toBe(true); @@ -66,10 +63,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('test'))).toBe(true); @@ -90,10 +84,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('devicectl') && c.includes('launch'))).toBe(true); @@ -113,10 +104,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { processId: 12345 }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('devicectl') && c.includes('terminate'))).toBe( @@ -138,10 +126,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { appPath: '/path/to/MyApp.app' }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('devicectl') && c.includes('install'))).toBe(true); @@ -162,10 +147,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( @@ -180,10 +162,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); expect(harness.capturedCommands.length).toBeGreaterThan(0); }); @@ -197,10 +176,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { workspaceRoot: '/path/to/workspace' }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('get_app_bundle_id responds with content', async () => { @@ -210,10 +186,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { appPath: '/path/to/MyApp.app' }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('get_mac_bundle_id responds with content', async () => { @@ -223,10 +196,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { appPath: '/path/to/MyApp.app' }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); }); @@ -246,10 +216,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('MyMacApp'))).toBe( @@ -272,10 +239,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild'))).toBe(true); @@ -296,10 +260,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('test'))).toBe(true); @@ -312,10 +273,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { appPath: '/path/to/MyMacApp.app' }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('stop_mac_app captures kill command with processId', async () => { @@ -325,10 +283,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { processId: 54321 }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('kill') && c.includes('54321'))).toBe(true); @@ -341,10 +296,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: { appName: 'MyMacApp' }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('MyMacApp'))).toBe(true); @@ -365,10 +317,7 @@ describe('MCP Device and macOS Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( diff --git a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts index c0d9680a..4304b44b 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { getContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -23,16 +24,12 @@ describe('MCP Doctor Tool (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); + const content = getContent(result); expect(content.length).toBeGreaterThan(0); - const hasText = - Array.isArray(content) && - content.some( - (c) => 'text' in c && typeof c.text === 'string' && c.text.includes('XcodeBuildMCP Doctor'), - ); + const hasText = content.some( + (c) => typeof c.text === 'string' && c.text.includes('XcodeBuildMCP Doctor'), + ); expect(hasText).toBe(true); }); }); diff --git a/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts b/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts index f7ee4343..2f6f75f8 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-invocation.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { isErrorResponse, expectContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -76,10 +77,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('simctl') && c.includes('list'))).toBe(true); @@ -102,10 +100,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('MyApp'))).toBe(true); @@ -126,10 +121,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild') && c.includes('clean'))).toBe(true); @@ -144,10 +136,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('swift') && c.includes('build'))).toBe(true); @@ -167,10 +156,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('simctl') && c.includes('boot'))).toBe(true); @@ -190,10 +176,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); expect(harness.capturedCommands.length).toBeGreaterThan(0); }); @@ -205,10 +188,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); expect(harness.capturedCommands.length).toBeGreaterThan(0); }); @@ -222,10 +202,7 @@ describe('MCP Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('session_show_defaults returns current defaults', async () => { @@ -234,10 +211,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('show_build_settings responds with content', async () => { @@ -255,10 +229,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); }); @@ -276,22 +247,7 @@ describe('MCP Tool Invocation (e2e)', () => { arguments: {}, }); - const content = 'content' in result ? result.content : []; - const isError = 'isError' in result ? result.isError : false; - const hasErrorText = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && - typeof c.text === 'string' && - (c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('required') || - c.text.toLowerCase().includes('must provide') || - c.text.toLowerCase().includes('invalid') || - c.text.toLowerCase().includes('missing') || - c.text.toLowerCase().includes('provide')), - ); - expect(isError || hasErrorText).toBe(true); + expect(isErrorResponse(result)).toBe(true); }); it('returns error response for non-existent tool', async () => { @@ -304,16 +260,7 @@ describe('MCP Tool Invocation (e2e)', () => { name: 'this_tool_does_not_exist', arguments: {}, }); - // If it didn't throw, check for error in the response - const isError = 'isError' in result ? result.isError : false; - const content = 'content' in result ? result.content : []; - const hasError = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && typeof c.text === 'string' && c.text.toLowerCase().includes('error'), - ); - expect(isError || hasError).toBe(true); + expect(isErrorResponse(result)).toBe(true); } catch (err: unknown) { threw = true; errorMessage = (err as Error).message; diff --git a/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts b/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts index 3df55c01..f7b628cd 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-logging.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { isErrorResponse, expectContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -34,10 +35,7 @@ describe('MCP Logging Tools (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('stop_sim_log_cap returns error for unknown session', async () => { @@ -49,23 +47,8 @@ describe('MCP Logging Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); - - const isError = 'isError' in result ? result.isError : false; - const hasErrorText = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && - typeof c.text === 'string' && - (c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('not found') || - c.text.toLowerCase().includes('invalid')), - ); - expect(isError || hasErrorText).toBe(true); + expectContent(result); + expect(isErrorResponse(result)).toBe(true); }); it('start_device_log_cap requires deviceId and bundleId via session', async () => { @@ -83,10 +66,7 @@ describe('MCP Logging Tools (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('stop_device_log_cap returns error for unknown session', async () => { @@ -98,22 +78,7 @@ describe('MCP Logging Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); - - const isError = 'isError' in result ? result.isError : false; - const hasErrorText = - Array.isArray(content) && - content.some( - (c) => - 'text' in c && - typeof c.text === 'string' && - (c.text.toLowerCase().includes('error') || - c.text.toLowerCase().includes('not found') || - c.text.toLowerCase().includes('failed')), - ); - expect(isError || hasErrorText).toBe(true); + expectContent(result); + expect(isErrorResponse(result)).toBe(true); }); }); diff --git a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts index 811d868f..bac52c71 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { expectContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -24,10 +25,7 @@ describe('MCP Project Scaffolding Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('scaffold_macos_project returns content with valid args', async () => { @@ -40,9 +38,6 @@ describe('MCP Project Scaffolding Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); }); diff --git a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts index 43bfb334..6789b6e8 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; -import { extractText } from '../test-helpers.ts'; +import { extractText, expectContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -136,10 +136,7 @@ describe('MCP Session Management (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); // The captured commands should include the session default scheme const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); diff --git a/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts b/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts index bf5e335b..8d433d6a 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-simulator.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { expectContent } from '../test-helpers.ts'; const SIM_UUID = 'AAAAAAAA-1111-4222-A333-444444444444'; @@ -94,10 +95,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild'))).toBe(true); @@ -112,10 +110,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('xcodebuild'))).toBe(true); @@ -130,10 +125,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('simctl') && c.includes('launch'))).toBe(true); @@ -148,10 +140,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('simctl') && c.includes('terminate'))).toBe(true); @@ -168,10 +157,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('open_sim captures open command', async () => { @@ -181,10 +167,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('open') && c.includes('Simulator'))).toBe(true); @@ -201,10 +184,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); }); it('get_sim_app_path captures xcodebuild -showBuildSettings command', async () => { @@ -218,10 +198,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( @@ -240,10 +217,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( @@ -264,10 +238,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('simctl') && c.includes('erase'))).toBe(true); @@ -284,10 +255,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( @@ -307,10 +275,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( @@ -329,10 +294,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( @@ -353,10 +315,7 @@ describe('MCP Simulator Tool Invocation (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect( diff --git a/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts b/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts index 23479654..5caecc97 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-swift-package.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createMcpTestHarness, type McpTestHarness } from '../mcp-test-harness.ts'; +import { expectContent } from '../test-helpers.ts'; let harness: McpTestHarness; @@ -29,10 +30,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('swift') && c.includes('clean'))).toBe(true); @@ -47,10 +45,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('swift') && c.includes('test'))).toBe(true); @@ -65,10 +60,7 @@ describe('MCP Swift Package Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); + expectContent(result); const commandStrs = harness.capturedCommands.map((c) => c.command.join(' ')); expect(commandStrs.some((c) => c.includes('swift') && c.includes('run'))).toBe(true); @@ -83,15 +75,8 @@ describe('MCP Swift Package Tools (e2e)', () => { }, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); - - const hasText = - Array.isArray(content) && - content.some((c) => 'text' in c && typeof c.text === 'string' && c.text.length > 0); - expect(hasText).toBe(true); + const content1 = expectContent(result); + expect(content1.some((c) => typeof c.text === 'string' && c.text.length > 0)).toBe(true); }); it('swift_package_list returns content listing processes', async () => { @@ -101,14 +86,7 @@ describe('MCP Swift Package Tools (e2e)', () => { arguments: {}, }); - expect(result).toBeDefined(); - const content = 'content' in result ? result.content : []; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThan(0); - - const hasText = - Array.isArray(content) && - content.some((c) => 'text' in c && typeof c.text === 'string' && c.text.length > 0); - expect(hasText).toBe(true); + const content2 = expectContent(result); + expect(content2.some((c) => typeof c.text === 'string' && c.text.length > 0)).toBe(true); }); }); diff --git a/src/smoke-tests/mcp-test-harness.ts b/src/smoke-tests/mcp-test-harness.ts index 0251c986..25c1f549 100644 --- a/src/smoke-tests/mcp-test-harness.ts +++ b/src/smoke-tests/mcp-test-harness.ts @@ -71,11 +71,13 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis if (opts?.commandResponses) { const commandStr = command.join(' '); - for (const [pattern, response] of Object.entries(opts.commandResponses)) { + const sorted = Object.entries(opts.commandResponses).sort(([a], [b]) => b.length - a.length); + for (const [pattern, response] of sorted) { if (commandStr.includes(pattern)) { return { ...defaultCommandResponse, ...response, + exitCode: response.success ? 0 : 1, }; } } @@ -90,9 +92,11 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis __resetToolRegistryForTests(); sessionStore.clear(); + const mockFs = createMockFileSystemExecutor(); + // Set executor overrides on the vitest-resolved source modules __setTestCommandExecutorOverride(capturingExecutor); - __setTestFileSystemExecutorOverride(createMockFileSystemExecutor()); + __setTestFileSystemExecutorOverride(mockFs); // Also set overrides on the built module instances (used by dynamically imported tool handlers) const buildRoot = resolve(getPackageRoot(), 'build'); @@ -111,7 +115,7 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis __clearTestExecutorOverrides: typeof __clearTestExecutorOverrides; }; builtCommandModule.__setTestCommandExecutorOverride(capturingExecutor); - builtCommandModule.__setTestFileSystemExecutorOverride(createMockFileSystemExecutor()); + builtCommandModule.__setTestFileSystemExecutorOverride(mockFs); // Set debugger tool context override (source module) __setTestDebuggerToolContextOverride({ @@ -141,7 +145,6 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis initConfigStore: typeof initConfigStore; }; builtConfigStoreModule.__resetConfigStoreForTests(); - const mockFs = createMockFileSystemExecutor(); await builtConfigStoreModule.initConfigStore({ cwd: '/test', fs: mockFs, @@ -189,7 +192,7 @@ export async function createMcpTestHarness(opts?: McpTestHarnessOptions): Promis debug: true, disableXcodeAutoSync: true, }, - fileSystemExecutor: createMockFileSystemExecutor(), + fileSystemExecutor: mockFs, }); // Create InMemoryTransport linked pair diff --git a/src/smoke-tests/test-helpers.ts b/src/smoke-tests/test-helpers.ts index 0371f2e7..db9ac27d 100644 --- a/src/smoke-tests/test-helpers.ts +++ b/src/smoke-tests/test-helpers.ts @@ -15,7 +15,6 @@ export function isErrorResponse(result: unknown): boolean { c.text.toLowerCase().includes('required') || c.text.toLowerCase().includes('missing') || c.text.toLowerCase().includes('must provide') || - c.text.toLowerCase().includes('provide') || c.text.toLowerCase().includes('fail')), ); } @@ -24,3 +23,11 @@ export function getContent(result: unknown): Array<{ type?: string; text?: strin const r = result as { content?: Array<{ type?: string; text?: string }> }; return Array.isArray(r.content) ? r.content : []; } + +export function expectContent(result: unknown): Array<{ type?: string; text?: string }> { + const content = getContent(result); + if (content.length === 0) { + throw new Error('Expected result to have non-empty content'); + } + return content; +} diff --git a/src/utils/tool-registry.ts b/src/utils/tool-registry.ts index 46024567..527d68ab 100644 --- a/src/utils/tool-registry.ts +++ b/src/utils/tool-registry.ts @@ -171,7 +171,11 @@ export async function updateWorkflowsFromManifest( export function __resetToolRegistryForTests(): void { for (const tool of registryState.tools.values()) { - tool.remove(); + try { + tool.remove(); + } catch { + // Safe to ignore: server may already be closed during cleanup + } } registryState.tools.clear(); registryState.enabledWorkflows.clear(); From 4f53f749f6f84a6a0d49c0a0f9484144fc9d37f7 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 7 Feb 2026 13:03:47 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=90=9B=20fix(smoke-tests):=20add=20ex?= =?UTF-8?q?plicit=20env=20type=20to=20satisfy=20strict=20typecheck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spread of process.env narrows the type so VITEST and NODE_ENV are not known properties. Use Record to allow delete on arbitrary keys. --- src/smoke-tests/__tests__/cli-surface.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smoke-tests/__tests__/cli-surface.test.ts b/src/smoke-tests/__tests__/cli-surface.test.ts index 92896bc4..e086698b 100644 --- a/src/smoke-tests/__tests__/cli-surface.test.ts +++ b/src/smoke-tests/__tests__/cli-surface.test.ts @@ -4,7 +4,7 @@ import { resolve } from 'path'; const CLI = resolve(__dirname, '../../../build/cli.js'); const cliEnv = (() => { - const env = { ...process.env, NO_COLOR: '1' }; + const env: Record = { ...process.env, NO_COLOR: '1' }; // Remove test environment markers so the CLI binary runs in production mode delete env.VITEST; delete env.NODE_ENV; From 5092733718e22e131bf351dc303804dee8055863 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 7 Feb 2026 16:08:59 +0000 Subject: [PATCH 5/5] test(smoke-tests): Harden CLI invocation and teardown assertions Use execFileSync for CLI smoke invocations to avoid shell-string execution and satisfy code scanning feedback. Require non-empty output in the --output json smoke test even when the tool exits non-zero on non-macOS. Guard harness cleanup in doctor and scaffolding suites so setup failures do not mask root causes. Use distinctive session defaults in clear-defaults assertions to avoid fragile substring matches. Co-Authored-By: Claude --- src/smoke-tests/__tests__/cli-surface.test.ts | 13 ++++++++----- src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts | 2 +- .../__tests__/e2e-mcp-scaffolding.test.ts | 2 +- src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts | 6 +++--- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/smoke-tests/__tests__/cli-surface.test.ts b/src/smoke-tests/__tests__/cli-surface.test.ts index e086698b..cfbea240 100644 --- a/src/smoke-tests/__tests__/cli-surface.test.ts +++ b/src/smoke-tests/__tests__/cli-surface.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { execSync } from 'child_process'; +import { execFileSync } from 'child_process'; import { resolve } from 'path'; const CLI = resolve(__dirname, '../../../build/cli.js'); @@ -11,7 +11,8 @@ const cliEnv = (() => { return env; })(); const run = (args: string): string => { - return execSync(`node ${CLI} ${args}`, { + const argv = args.trim() ? args.trim().split(/\s+/) : []; + return execFileSync('node', [CLI, ...argv], { encoding: 'utf8', timeout: 15_000, env: cliEnv, @@ -108,12 +109,14 @@ describe('CLI Surface (e2e)', () => { // list_sims is a good candidate -- it will fail to run xcrun but should // return structured JSON output even on error const result = runMayFail('simulator list-sims --output json'); - // Even if the tool fails (no xcrun), the output format should be JSON + const output = result.stdout.trim(); + expect(output.length).toBeGreaterThan(0); + // Even if the tool fails (no xcrun), a successful run should be JSON if (result.status === 0) { - const parsed = JSON.parse(result.stdout); + const parsed = JSON.parse(output); expect(parsed).toBeDefined(); } - // If it fails, that's also acceptable on non-macOS platforms + // If it fails, that's acceptable on non-macOS platforms as long as output is present }); it('missing required args produces user-friendly error', () => { diff --git a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts index 4304b44b..23eb1359 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-doctor.test.ts @@ -13,7 +13,7 @@ beforeAll(async () => { }, 30_000); afterAll(async () => { - await harness.cleanup(); + await harness?.cleanup(); }); describe('MCP Doctor Tool (e2e)', () => { diff --git a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts index bac52c71..6e489a71 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-scaffolding.test.ts @@ -11,7 +11,7 @@ beforeAll(async () => { }, 30_000); afterAll(async () => { - await harness.cleanup(); + await harness?.cleanup(); }); describe('MCP Project Scaffolding Tools (e2e)', () => { diff --git a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts index 6789b6e8..401465ab 100644 --- a/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts +++ b/src/smoke-tests/__tests__/e2e-mcp-sessions.test.ts @@ -73,7 +73,7 @@ describe('MCP Session Management (e2e)', () => { // Set defaults await harness.client.callTool({ name: 'session_set_defaults', - arguments: { scheme: 'App', projectPath: '/proj' }, + arguments: { scheme: 'ClearMeScheme', projectPath: '/clear-me-proj' }, }); // Clear all @@ -90,8 +90,8 @@ describe('MCP Session Management (e2e)', () => { const text = extractText(result); // Should not contain the previously set values - expect(text).not.toContain('App'); - expect(text).not.toContain('/proj'); + expect(text).not.toContain('ClearMeScheme'); + expect(text).not.toContain('/clear-me-proj'); }); it('session_clear_defaults clears specific keys', async () => {