diff --git a/package.json b/package.json index 5572f94b1..c1d92d990 100644 --- a/package.json +++ b/package.json @@ -159,6 +159,7 @@ "!android-snapshot-helper/dist/*.idsig", "android-multitouch-helper/dist", "!android-multitouch-helper/dist/*.idsig", + "third_party/serve-sim-camera", "src/platforms/linux/atspi-dump.py", "skills", "server.json", diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index ea5e249d5..efa5d20f6 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -66,12 +66,14 @@ test('apps.open resolves session device identifiers from open response', async ( const result = await client.apps.open({ app: 'Settings', platform: 'ios', + cameraVideo: './fixtures/back.mp4', relaunch: true, }); assert.equal(setup.calls.length, 1); assert.equal(setup.calls[0]?.command, 'open'); assert.deepEqual(setup.calls[0]?.positionals, ['Settings']); + assert.equal(setup.calls[0]?.flags?.cameraVideo, './fixtures/back.mp4'); assert.equal(result.identifiers.session, 'qa'); assert.equal(result.identifiers.deviceId, 'SIM-001'); assert.equal(result.identifiers.udid, 'SIM-001'); @@ -80,6 +82,35 @@ test('apps.open resolves session device identifiers from open response', async ( assert.equal(result.device?.ios?.simulatorSetPath, '/tmp/sim-set'); }); +test('devices.boot forwards Android emulator camera options', async () => { + const setup = createTransport(async () => ({ + ok: true, + data: { + platform: 'android', + target: 'mobile', + device: 'Pixel_9_Pro_XL', + id: 'emulator-5554', + kind: 'emulator', + booted: true, + }, + })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.devices.boot({ + platform: 'android', + device: 'Pixel_9_Pro_XL', + cameraFront: '/tmp/front.mp4', + cameraBack: 'virtualscene', + }); + + assert.equal(setup.calls.length, 1); + assert.equal(setup.calls[0]?.command, 'boot'); + assert.equal(setup.calls[0]?.flags?.platform, 'android'); + assert.equal(setup.calls[0]?.flags?.device, 'Pixel_9_Pro_XL'); + assert.equal(setup.calls[0]?.flags?.cameraFront, '/tmp/front.mp4'); + assert.equal(setup.calls[0]?.flags?.cameraBack, 'virtualscene'); +}); + test('apps.open forwards explicit runtime hints through the daemon request', async () => { const setup = createTransport(async () => ({ ok: true, diff --git a/src/backend.ts b/src/backend.ts index 3f98026de..88e85806e 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -194,6 +194,7 @@ export type BackendOpenTarget = { }; export type BackendOpenOptions = { + cameraVideo?: string; launchArgs?: string[]; relaunch?: boolean; }; diff --git a/src/client-normalizers.ts b/src/client-normalizers.ts index 25cd3d796..d3c88661a 100644 --- a/src/client-normalizers.ts +++ b/src/client-normalizers.ts @@ -277,6 +277,7 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { androidDeviceAllowlist: options.androidDeviceAllowlist, surface: options.surface, activity: options.activity, + cameraVideo: options.cameraVideo, launchConsole: options.launchConsole, launchArgs: options.launchArgs, relaunch: options.relaunch, @@ -311,6 +312,8 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { pauseMs: options.pauseMs, pattern: options.pattern, headless: options.headless, + cameraFront: options.cameraFront, + cameraBack: options.cameraBack, restart: options.restart, replayUpdate: options.replayUpdate, replayBackend: options.replayBackend, diff --git a/src/client-types.ts b/src/client-types.ts index ed9ea1bd1..0f93b6f29 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -189,6 +189,7 @@ export type AppOpenOptions = AgentDeviceRequestOverrides & url?: string; surface?: SessionSurface; activity?: string; + cameraVideo?: string; launchConsole?: string; launchArgs?: string[]; relaunch?: boolean; @@ -555,6 +556,9 @@ type RepeatedPressOptions = { export type DeviceBootOptions = DeviceCommandBaseOptions & { headless?: boolean; + cameraVideo?: string; + cameraFront?: string; + cameraBack?: string; }; export type DeviceShutdownOptions = DeviceCommandBaseOptions; @@ -850,6 +854,8 @@ type CommandExecutionOptions = Partial & { pauseMs?: number; pattern?: SwipePattern; headless?: boolean; + cameraFront?: string; + cameraBack?: string; restart?: boolean; replayUpdate?: boolean; replayBackend?: string; @@ -878,6 +884,7 @@ export type InternalRequestOptions = AgentDeviceClientConfig & overlayRefs?: boolean; surface?: SessionSurface; activity?: string; + cameraVideo?: string; launchConsole?: string; launchArgs?: string[]; relaunch?: boolean; diff --git a/src/commands/apps.ts b/src/commands/apps.ts index 25833982d..bf3c54696 100644 --- a/src/commands/apps.ts +++ b/src/commands/apps.ts @@ -25,6 +25,7 @@ const MAX_APP_PUSH_PAYLOAD_BYTES = 8 * 1024; export type OpenAppCommandOptions = CommandContext & BackendOpenTarget & { + cameraVideo?: string; launchArgs?: string[]; relaunch?: boolean; }; @@ -105,6 +106,7 @@ export const openAppCommand: RuntimeCommand ({ ...commonInputFromFlags(flags), headless: flags.headless, + cameraFront: flags.cameraFront, + cameraBack: flags.cameraBack, }), shutdown: (_positionals, flags) => commonInputFromFlags(flags), prepare: (positionals, flags) => ({ @@ -41,6 +43,7 @@ export const appCliReaders = { url: positionals[1], surface: flags.surface, activity: flags.activity, + cameraVideo: flags.cameraVideo, launchConsole: flags.launchConsole, launchArgs: flags.launchArgs, relaunch: flags.relaunch, diff --git a/src/commands/client-command-metadata.ts b/src/commands/client-command-metadata.ts index 718a232d8..7815586a2 100644 --- a/src/commands/client-command-metadata.ts +++ b/src/commands/client-command-metadata.ts @@ -37,6 +37,8 @@ export const clientCommandMetadata = [ defineClientCommandMetadata('devices', {}), defineClientCommandMetadata('boot', { headless: booleanField('Boot without showing simulator UI when supported.'), + cameraFront: stringField('Android emulator front camera mode or video file path used at boot.'), + cameraBack: stringField('Android emulator back camera mode or video file path used at boot.'), }), defineClientCommandMetadata('shutdown', {}), defineClientCommandMetadata('prepare', { @@ -54,6 +56,7 @@ export const clientCommandMetadata = [ url: stringField('Optional URL passed with an app shell.'), surface: enumField(SESSION_SURFACES), activity: stringField('Android activity name.'), + cameraVideo: stringField('iOS simulator video file path injected as the app camera stream.'), launchConsole: stringField('Launch console mode.'), launchArgs: stringArrayField( 'Launch arguments forwarded verbatim to the platform launch command.', diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index 9c18cac26..4ee7fef3f 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -77,6 +77,52 @@ test('dispatch open rejects launch arguments without an app target', async () => ); }); +test('dispatch open rejects camera video without an app target', async () => { + await assert.rejects( + () => dispatchCommand(IOS_SIMULATOR, 'open', [], undefined, { cameraVideo: './back.mp4' }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'INVALID_ARGS'); + assert.match((error as AppError).message, /requires an app target/i); + return true; + }, + ); +}); + +test('dispatch open forwards iOS simulator camera video to openIosApp', async () => { + await dispatchCommand(IOS_SIMULATOR, 'open', ['com.example.app'], undefined, { + cameraVideo: '/tmp/back.mp4', + }); + + assert.equal(mockOpenIosApp.mock.calls.length, 1); + assert.equal(mockOpenIosApp.mock.calls[0]?.[0], IOS_SIMULATOR); + assert.equal(mockOpenIosApp.mock.calls[0]?.[1], 'com.example.app'); + assert.equal(mockOpenIosApp.mock.calls[0]?.[2]?.cameraVideo, '/tmp/back.mp4'); +}); + +test('dispatch open rejects camera video outside iOS simulator', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await assert.rejects( + () => + dispatchCommand(device, 'open', ['com.example.app'], undefined, { + cameraVideo: './back.mp4', + }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + assert.match((error as AppError).message, /iOS simulators/i); + return true; + }, + ); +}); + test('dispatch open forwards Android launch arguments to openAndroidApp', async () => { const device: DeviceInfo = { platform: 'android', diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 6ff1d58b5..d71e6e95f 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -33,6 +33,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { requestId?: string; appBundleId?: string; activity?: string; + cameraVideo?: string; launchConsole?: string; launchArgs?: string[]; clearAppState?: boolean; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 6143967d9..943929d9d 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -175,6 +175,7 @@ async function handleOpenCommand( ): Promise> { const app = positionals[0]; const url = positionals[1]; + const cameraVideo = context?.cameraVideo; const launchConsole = context?.launchConsole; const launchArgs = context?.launchArgs; if (positionals.length > 2) { @@ -187,9 +188,18 @@ async function handleOpenCommand( if (launchArgs && launchArgs.length > 0) { throw new AppError('INVALID_ARGS', '--launch-args requires an app target'); } + if (cameraVideo) { + throw new AppError('INVALID_ARGS', '--camera-video requires an app target'); + } await interactor.openDevice(); return { app: null, ...successText('Opened device') }; } + if (cameraVideo && (device.platform !== 'ios' || device.kind !== 'simulator')) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--camera-video is supported only for iOS simulators.', + ); + } if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) { throw new AppError('UNSUPPORTED_OPERATION', LAUNCH_CONSOLE_IOS_SIMULATOR_ONLY_MESSAGE); } @@ -212,6 +222,7 @@ async function handleOpenCommand( await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, + cameraVideo, launchArgs, url, }); @@ -220,6 +231,9 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (cameraVideo && isDeepLinkTarget(app)) { + throw new AppError('INVALID_ARGS', '--camera-video requires an app target'); + } if (context?.clearAppState) { if (isDeepLinkTarget(app)) { throw new AppError( @@ -232,6 +246,7 @@ async function handleOpenCommand( await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, + cameraVideo, launchConsole, launchArgs, }); diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 9966d2cca..a84a2edf6 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -57,6 +57,7 @@ export type Interactor = { options?: { activity?: string; appBundleId?: string; + cameraVideo?: string; launchConsole?: string; launchArgs?: string[]; url?: string; diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index 744adcb01..82cb79b56 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -29,6 +29,7 @@ export function createAppleInteractor( open: (app, options) => openIosApp(device, app, { appBundleId: options?.appBundleId, + cameraVideo: options?.cameraVideo, launchConsole: options?.launchConsole, launchArgs: options?.launchArgs, url: options?.url, diff --git a/src/daemon-client.ts b/src/daemon-client.ts index 981b8b815..20cad81d8 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -54,6 +54,7 @@ export type OpenAppOptions = { udid?: NonNullable['udid']; serial?: NonNullable['serial']; activity?: NonNullable['activity']; + cameraVideo?: NonNullable['cameraVideo']; launchConsole?: NonNullable['launchConsole']; launchArgs?: NonNullable['launchArgs']; out?: NonNullable['out']; @@ -226,6 +227,7 @@ export async function openApp(options: OpenAppOptions = {}): Promise { assert.equal(context.clearAppState, true); }); +test('contextFromFlags forwards iOS simulator camera video path', () => { + const flags: CommandFlags = { cameraVideo: './fixtures/camera-feed.mp4' }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.cameraVideo, './fixtures/camera-feed.mp4'); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/context.ts b/src/daemon/context.ts index ff35ded64..d490ea474 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -23,6 +23,7 @@ export function contextFromFlags( requestId: effectiveRequestId, appBundleId, activity: flags?.activity, + cameraVideo: flags?.cameraVideo, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, clearAppState: flags?.clearAppState, diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index bd611040b..0742e4d83 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -937,6 +937,61 @@ test('boot launches Android emulator with GUI when no running device matches', a } }); +test('boot launches Android emulator with camera video files', async () => { + const sessionStore = makeSessionStore(); + mockResolveTargetDevice.mockRejectedValue(new AppError('DEVICE_NOT_FOUND', 'No device found')); + const launchCalls: Array<{ + avdName: string; + serial?: string; + headless?: boolean; + cameraFront?: string; + cameraBack?: string; + }> = []; + mockEnsureAndroidEmulatorBooted.mockImplementation( + async ({ avdName, serial, headless, cameraFront, cameraBack }) => { + launchCalls.push({ avdName, serial, headless, cameraFront, cameraBack }); + return { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel_9_Pro_XL', + kind: 'emulator', + target: 'mobile', + booted: true, + }; + }, + ); + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'boot', + positionals: [], + flags: { + platform: 'android', + device: 'Pixel_9_Pro_XL', + cameraFront: '/tmp/front.mp4', + cameraBack: '/tmp/back.mp4', + }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(true); + expect(launchCalls).toEqual([ + { + avdName: 'Pixel_9_Pro_XL', + serial: undefined, + headless: false, + cameraFront: '/tmp/front.mp4', + cameraBack: '/tmp/back.mp4', + }, + ]); +}); + test('boot --headless requires avd selector when device cannot be resolved', async () => { const sessionStore = makeSessionStore(); mockResolveTargetDevice.mockRejectedValue(new AppError('DEVICE_NOT_FOUND', 'No device found')); diff --git a/src/daemon/handlers/session-open.ts b/src/daemon/handlers/session-open.ts index 22d298e28..d9ccbb380 100644 --- a/src/daemon/handlers/session-open.ts +++ b/src/daemon/handlers/session-open.ts @@ -101,11 +101,26 @@ function contextForRuntimeLaunchUrl( traceLogPath?: string, ): ReturnType { const context = contextFromFlags(logPath, flags, appBundleId, traceLogPath); + delete context.cameraVideo; delete context.launchConsole; delete context.launchArgs; return context; } +function contextForOpenDispatch( + logPath: string, + flags: DaemonRequest['flags'], + appBundleId: string | undefined, + traceLogPath: string | undefined, + cwd: string | undefined, +): ReturnType { + const context = contextFromFlags(logPath, flags, appBundleId, traceLogPath); + if (context.cameraVideo) { + context.cameraVideo = SessionStore.expandHome(context.cameraVideo, cwd); + } + return context; +} + function buildStartupPerfSample( startedAtMs: number, appTarget: string | undefined, @@ -218,7 +233,7 @@ async function completeOpenCommand(params: { } const openDispatchSession = provisionalSession.session ?? existingSession; await dispatchCommand(device, 'open', openPositionals, req.flags?.out, { - ...contextFromFlags(logPath, req.flags, sessionAppBundleId), + ...contextForOpenDispatch(logPath, req.flags, sessionAppBundleId, traceLogPath, req.meta?.cwd), }); timing.openDispatchDurationMs = Math.max(0, Date.now() - openStartedAtMs); const launchUrlStartedAtMs = Date.now(); diff --git a/src/daemon/handlers/session-state.ts b/src/daemon/handlers/session-state.ts index 78c465bda..af92b436f 100644 --- a/src/daemon/handlers/session-state.ts +++ b/src/daemon/handlers/session-state.ts @@ -18,6 +18,8 @@ async function ensureAndroidEmulatorBoot(params: { avdName: string; serial?: string; headless?: boolean; + cameraFront?: string; + cameraBack?: string; }): Promise { const { ensureAndroidEmulatorBooted } = await import('../../platforms/android/devices.ts'); return await ensureAndroidEmulatorBooted(params); @@ -148,12 +150,19 @@ export async function handleSessionStateCommands(params: { normalizePlatformSelector(flags.platform) ?? session?.device.platform; const targetsAndroid = normalizedPlatform === 'android'; const wantsAndroidHeadless = flags.headless === true; + const wantsAndroidCamera = Boolean(flags.cameraFront || flags.cameraBack); if (wantsAndroidHeadless && !targetsAndroid) { return errorResponse( 'INVALID_ARGS', 'boot --headless is supported only for Android emulators.', ); } + if (wantsAndroidCamera && !targetsAndroid) { + return errorResponse( + 'INVALID_ARGS', + 'boot --camera-front/--camera-back is supported only for Android emulators.', + ); + } const fallbackAvdName = resolveAndroidEmulatorAvdName({ flags, @@ -173,13 +182,13 @@ export async function handleSessionStateCommands(params: { const appErr = asAppError(error); if ( targetsAndroid && - wantsAndroidHeadless && + (wantsAndroidHeadless || wantsAndroidCamera) && !fallbackAvdName && appErr.code === 'DEVICE_NOT_FOUND' ) { return errorResponse( 'INVALID_ARGS', - 'boot --headless requires --device (or an Android emulator session target).', + androidBootOptionsRequireDeviceMessage(wantsAndroidCamera), ); } if ( @@ -193,6 +202,8 @@ export async function handleSessionStateCommands(params: { avdName: fallbackAvdName, serial: flags.serial, headless: wantsAndroidHeadless, + cameraFront: typeof flags.cameraFront === 'string' ? flags.cameraFront : undefined, + cameraBack: typeof flags.cameraBack === 'string' ? flags.cameraBack : undefined, }); launchedAndroidEmulator = true; } @@ -204,11 +215,13 @@ export async function handleSessionStateCommands(params: { ); } - if (targetsAndroid && wantsAndroidHeadless) { + if (targetsAndroid && (wantsAndroidHeadless || wantsAndroidCamera)) { if (device.platform !== 'android' || device.kind !== 'emulator') { return errorResponse( 'INVALID_ARGS', - 'boot --headless is supported only for Android emulators.', + wantsAndroidCamera + ? 'boot --camera-front/--camera-back is supported only for Android emulators.' + : 'boot --headless is supported only for Android emulators.', ); } if (!launchedAndroidEmulator) { @@ -220,13 +233,15 @@ export async function handleSessionStateCommands(params: { if (!avdName) { return errorResponse( 'INVALID_ARGS', - 'boot --headless requires --device (or an Android emulator session target).', + androidBootOptionsRequireDeviceMessage(wantsAndroidCamera), ); } device = await ensureAndroidEmulatorBoot({ avdName, serial: flags.serial, - headless: true, + headless: wantsAndroidHeadless, + cameraFront: typeof flags.cameraFront === 'string' ? flags.cameraFront : undefined, + cameraBack: typeof flags.cameraBack === 'string' ? flags.cameraBack : undefined, }); } await ensureDeviceReady(device); @@ -332,6 +347,12 @@ export async function handleSessionStateCommands(params: { return null; } +function androidBootOptionsRequireDeviceMessage(wantsAndroidCamera: boolean): string { + return wantsAndroidCamera + ? 'boot --camera-front/--camera-back requires --device (or an Android emulator session target).' + : 'boot --headless requires --device (or an Android emulator session target).'; +} + function shutdownFailureMessage( shutdown: Awaited>, ): string { diff --git a/src/platforms/android/__tests__/devices.test.ts b/src/platforms/android/__tests__/devices.test.ts index 1c2d58825..53b6587f6 100644 --- a/src/platforms/android/__tests__/devices.test.ts +++ b/src/platforms/android/__tests__/devices.test.ts @@ -355,6 +355,47 @@ test('ensureAndroidEmulatorBooted launches emulator with GUI by default', async }); }, 10_000); +test('ensureAndroidEmulatorBooted launches emulator with camera video files', async () => { + await withMockedAndroidTools(async ({ emulatorLogPath }) => { + const frontVideo = path.join(os.tmpdir(), 'front-camera.mp4'); + const backVideo = path.join(os.tmpdir(), 'back-camera.mp4'); + await fs.writeFile(frontVideo, 'front', 'utf8'); + await fs.writeFile(backVideo, 'back', 'utf8'); + + const device = await ensureAndroidEmulatorBooted({ + avdName: 'Pixel_9_Pro_XL', + timeoutMs: 5_000, + cameraFront: frontVideo, + cameraBack: backVideo, + }); + assert.equal(device.id, 'emulator-5554'); + const log = await fs.readFile(emulatorLogPath, 'utf8'); + assert.match( + log, + new RegExp(`-camera-front videofile:${escapeRegExp(path.resolve(frontVideo))}`), + ); + assert.match( + log, + new RegExp(`-camera-back videofile:${escapeRegExp(path.resolve(backVideo))}`), + ); + }); +}, 10_000); + +test('ensureAndroidEmulatorBooted rejects camera inputs for a running emulator', async () => { + await withMockedAndroidTools(async ({ emulatorBootedPath }) => { + await fs.writeFile(emulatorBootedPath, 'ready', 'utf8'); + await assert.rejects( + async () => + await ensureAndroidEmulatorBooted({ + avdName: 'Pixel_9_Pro_XL', + timeoutMs: 5_000, + cameraBack: '/tmp/back-camera.mp4', + }), + /camera inputs can only be applied when starting an emulator/, + ); + }); +}, 10_000); + test('ensureAndroidEmulatorBooted falls back to ANDROID_SDK_ROOT when PATH is incomplete', async () => { await withMockedAndroidSdkRoot(async ({ emulatorLogPath, sdkRoot }) => { const device = await ensureAndroidEmulatorBooted({ @@ -370,3 +411,7 @@ test('ensureAndroidEmulatorBooted falls back to ANDROID_SDK_ROOT when PATH is in assert.equal(process.env.ANDROID_HOME, sdkRoot); }); }, 10_000); + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/src/platforms/android/devices.ts b/src/platforms/android/devices.ts index 22d396460..36ebfea31 100644 --- a/src/platforms/android/devices.ts +++ b/src/platforms/android/devices.ts @@ -1,3 +1,5 @@ +import { existsSync } from 'node:fs'; +import path from 'node:path'; import { runCmd, runCmdDetached, whichCmd } from '../../utils/exec.ts'; import type { ExecResult } from '../../utils/exec.ts'; import { sleep } from '../../utils/timeouts.ts'; @@ -403,6 +405,8 @@ export async function ensureAndroidEmulatorBooted(params: { serial?: string; timeoutMs?: number; headless?: boolean; + cameraFront?: string; + cameraBack?: string; }): Promise { await ensureAndroidSdkPathConfigured(); const requestedAvdName = params.avdName.trim(); @@ -434,12 +438,9 @@ export async function ensureAndroidEmulatorBooted(params: { resolvedAvdName, params.serial, ); + assertCameraInputsCanApplyToEmulator(existing, resolvedAvdName, params); if (!existing) { - const launchArgs = ['-avd', resolvedAvdName]; - if (params.headless) { - launchArgs.push('-no-window', '-no-audio'); - } - runCmdDetached('emulator', launchArgs); + runCmdDetached('emulator', buildEmulatorLaunchArgs(resolvedAvdName, params)); } const discovered = @@ -468,6 +469,77 @@ export async function ensureAndroidEmulatorBooted(params: { }; } +function assertCameraInputsCanApplyToEmulator( + existing: DeviceInfo | undefined, + resolvedAvdName: string, + params: { + cameraFront?: string; + cameraBack?: string; + }, +): void { + if (!existing || (!params.cameraFront && !params.cameraBack)) return; + throw new AppError( + 'INVALID_STATE', + 'Android emulator camera inputs can only be applied when starting an emulator.', + { + avdName: resolvedAvdName, + serial: existing.id, + hint: 'Shut down the emulator first, then run boot again with --camera-front or --camera-back.', + }, + ); +} + +function buildEmulatorLaunchArgs( + resolvedAvdName: string, + params: { + headless?: boolean; + cameraFront?: string; + cameraBack?: string; + }, +): string[] { + const launchArgs = ['-avd', resolvedAvdName]; + if (params.headless) { + launchArgs.push('-no-window', '-no-audio'); + } + const cameraFront = resolveAndroidEmulatorCameraMode(params.cameraFront, 'front'); + if (cameraFront) launchArgs.push('-camera-front', cameraFront); + const cameraBack = resolveAndroidEmulatorCameraMode(params.cameraBack, 'back'); + if (cameraBack) launchArgs.push('-camera-back', cameraBack); + return launchArgs; +} + +function resolveAndroidEmulatorCameraMode( + value: string | undefined, + camera: 'front' | 'back', +): string | undefined { + const trimmed = value?.trim(); + if (!trimmed) return undefined; + if (trimmed.startsWith('videofile:')) { + const videoPath = trimmed.slice('videofile:'.length); + const resolvedPath = path.resolve(videoPath); + if (videoPath && existsSync(resolvedPath)) return `videofile:${resolvedPath}`; + } + if (isAndroidEmulatorCameraMode(trimmed, camera)) return trimmed; + const resolvedPath = path.resolve(trimmed); + if (existsSync(resolvedPath)) return `videofile:${resolvedPath}`; + throw new AppError( + 'INVALID_ARGS', + `Android emulator ${camera} camera input is not valid: ${trimmed}`, + { + hint: + camera === 'back' + ? 'Use a video file path, videofile:, emulated, virtualscene, webcam, or none.' + : 'Use a video file path, videofile:, emulated, webcam, or none.', + }, + ); +} + +function isAndroidEmulatorCameraMode(value: string, camera: 'front' | 'back'): boolean { + if (value === 'emulated' || value === 'none') return true; + if (/^webcam\d+$/.test(value)) return true; + return camera === 'back' && value === 'virtualscene'; +} + export async function waitForAndroidBoot(serial: string, timeoutMs = 60000): Promise { const timeoutBudget = timeoutMs; const deadline = Deadline.fromTimeoutMs(timeoutBudget); diff --git a/src/platforms/ios/__tests__/simulator-camera.test.ts b/src/platforms/ios/__tests__/simulator-camera.test.ts new file mode 100644 index 000000000..53882ec41 --- /dev/null +++ b/src/platforms/ios/__tests__/simulator-camera.test.ts @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { IOS_DEVICE, IOS_SIMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts'; +import { AppError } from '../../../utils/errors.ts'; +import { runCmdDetached } from '../../../utils/exec.ts'; +import { + prepareIosSimulatorCameraVideo, + stopIosSimulatorCameraVideo, +} from '../simulator-camera.ts'; + +vi.mock('../../../utils/exec.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runCmdDetached: vi.fn(() => 987_654), + }; +}); + +const mockRunCmdDetached = vi.mocked(runCmdDetached); + +afterEach(() => { + mockRunCmdDetached.mockClear(); +}); + +test('prepareIosSimulatorCameraVideo starts vendored helper and returns simctl child env', async () => { + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'agent-device-ios-camera-test-')); + const videoPath = path.join(tempDir, 'sample.mp4'); + await fsp.writeFile(videoPath, 'fixture'); + try { + const launch = await prepareIosSimulatorCameraVideo({ + device: IOS_SIMULATOR, + bundleId: 'com.example.camera', + videoPath, + }); + + assert.equal(mockRunCmdDetached.mock.calls.length, 1); + const [helperPath, helperArgs, helperOptions] = mockRunCmdDetached.mock.calls[0] ?? []; + assert.match(helperPath ?? '', /third_party\/serve-sim-camera\/bin\/camera-helper$/); + assert.deepEqual(helperArgs, [ + '--shm', + launch.shmName, + '--source', + 'video', + '--arg', + videoPath, + ]); + assert.equal(launch.helperPid, 987_654); + assert.equal(launch.videoPath, videoPath); + assert.match(launch.shmName, /^\/ad-camera-[a-f0-9]{12}$/); + assert.deepEqual(helperOptions?.stdio?.[0], 'ignore'); + assert.equal(typeof helperOptions?.stdio?.[1], 'number'); + assert.equal(helperOptions?.stdio?.[2], helperOptions?.stdio?.[1]); + assert.match( + launch.env.SIMCTL_CHILD_DYLD_INSERT_LIBRARIES ?? '', + /camera-injector\.dylib$/, + ); + const shmEnvKey = Object.keys(launch.env).find((key) => key.endsWith('_SHM_NAME')); + const mirrorEnvKey = Object.keys(launch.env).find((key) => key.endsWith('_MIRROR_MODE')); + assert.equal(launch.env[shmEnvKey ?? ''], launch.shmName); + assert.equal(launch.env[mirrorEnvKey ?? ''], 'auto'); + } finally { + await stopIosSimulatorCameraVideo(IOS_SIMULATOR, 'com.example.camera'); + await fsp.rm(tempDir, { force: true, recursive: true }); + } +}); + +test('prepareIosSimulatorCameraVideo rejects non-simulator devices', async () => { + await assert.rejects( + () => + prepareIosSimulatorCameraVideo({ + device: IOS_DEVICE, + bundleId: 'com.example.camera', + videoPath: '/tmp/sample.mp4', + }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + return true; + }, + ); +}); diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index d0d456cbf..b171b7aa3 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -51,6 +51,7 @@ import { buildSimctlArgsForDevice } from './simctl.ts'; import { runAppleToolCommand, runXcrun } from './tool-provider.ts'; import { prepareIosInstallArtifact } from './install-artifact.ts'; import { filterAppleAppsByBundlePrefix } from './app-filter.ts'; +import { prepareIosSimulatorCameraVideo, stopIosSimulatorCameraVideo } from './simulator-camera.ts'; import { closeMacOsApp, listMacApps, @@ -174,13 +175,26 @@ function parseUrlScheme(url: string): string | undefined { export async function openIosApp( device: DeviceInfo, app: string, - options?: { appBundleId?: string; launchConsole?: string; launchArgs?: string[]; url?: string }, + options?: { + appBundleId?: string; + launchConsole?: string; + launchArgs?: string[]; + cameraVideo?: string; + url?: string; + }, ): Promise { const launchConsole = options?.launchConsole?.trim(); const launchArgs = options?.launchArgs; + const cameraVideo = options?.cameraVideo?.trim(); if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) { throw new AppError('UNSUPPORTED_OPERATION', LAUNCH_CONSOLE_IOS_SIMULATOR_ONLY_MESSAGE); } + if (cameraVideo && (device.platform !== 'ios' || device.kind !== 'simulator')) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--camera-video is supported only for iOS simulators.', + ); + } if (device.platform === 'macos') { if (launchArgs && launchArgs.length > 0) { throw new AppError( @@ -203,6 +217,7 @@ export async function openIosApp( const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); await launchIosSimulatorApp(device, bundleId, { ...(launchArgs ? { launchArgs } : {}), + ...(cameraVideo ? { cameraVideo } : {}), }); await openIosSimulatorUrl(device, explicitUrl, undefined); return; @@ -224,6 +239,9 @@ export async function openIosApp( if (launchConsole) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (cameraVideo) { + throw new AppError('INVALID_ARGS', '--camera-video requires an app target.'); + } if (device.kind === 'simulator') { await openIosSimulatorUrl(device, deepLinkTarget, launchArgs); return; @@ -244,6 +262,7 @@ export async function openIosApp( await launchIosSimulatorApp(device, bundleId, { ...(launchConsole ? { launchConsole } : {}), ...(launchArgs ? { launchArgs } : {}), + ...(cameraVideo ? { cameraVideo } : {}), }); return; } @@ -283,20 +302,24 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { await ensureBootedSimulator(device); + const cameraLaunch = options?.cameraVideo + ? await prepareIosSimulatorCameraVideo({ + device, + bundleId, + videoPath: options.cameraVideo, + }) + : undefined; let consecutiveFBSFailures = 0; const MAX_CONSECUTIVE_FBS_FAILURES = 3; @@ -1111,9 +1141,10 @@ async function launchIosSimulatorApp( buildIosSimulatorLaunchArgs(device.id, bundleId, options), ); const result = options?.launchConsole - ? await runIosSimulatorConsoleLaunch(launchArgs, options.launchConsole) + ? await runIosSimulatorConsoleLaunch(launchArgs, options.launchConsole, cameraLaunch?.env) : await runXcrun(launchArgs, { allowFailure: true, + ...(cameraLaunch?.env ? { env: { ...process.env, ...cameraLaunch.env } } : {}), }); if (result.exitCode === 0) return; @@ -1139,6 +1170,9 @@ async function launchIosSimulatorApp( { deadline: launchDeadline }, ); } catch (error) { + if (cameraLaunch) { + await stopIosSimulatorCameraVideo(device, bundleId).catch(() => {}); + } if (isSimulatorLaunchFBSError(error)) { const appError = error as AppError; const probe = await probeSimulatorLaunchContext(device, bundleId); @@ -1152,10 +1186,11 @@ async function launchIosSimulatorApp( function buildIosSimulatorLaunchArgs( deviceId: string, bundleId: string, - options?: { launchConsole?: string; launchArgs?: string[] }, + options?: { launchConsole?: string; launchArgs?: string[]; cameraVideo?: string }, ): string[] { const args = ['launch']; if (options?.launchConsole) args.push('--console-pty'); + if (options?.cameraVideo) args.push('--terminate-running-process'); args.push(deviceId, bundleId); if (options?.launchArgs && options.launchArgs.length > 0) { args.push(...options.launchArgs); @@ -1166,12 +1201,14 @@ function buildIosSimulatorLaunchArgs( async function runIosSimulatorConsoleLaunch( launchArgs: string[], logPath: string, + env?: NodeJS.ProcessEnv, ): Promise>> { await fs.mkdir(path.dirname(logPath), { recursive: true }); try { const result = await runXcrun(launchArgs, { allowFailure: true, timeoutMs: IOS_SIMULATOR_CONSOLE_CAPTURE_MS, + ...(env ? { env: { ...process.env, ...env } } : {}), }); await writeIosSimulatorConsoleLog(logPath, result.stdout, result.stderr); return result; diff --git a/src/platforms/ios/simulator-camera.ts b/src/platforms/ios/simulator-camera.ts new file mode 100644 index 000000000..c8239f7b9 --- /dev/null +++ b/src/platforms/ios/simulator-camera.ts @@ -0,0 +1,191 @@ +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; +import { findProjectRoot } from '../../utils/version.ts'; +import { runCmdDetached } from '../../utils/exec.ts'; +import { AppError } from '../../utils/errors.ts'; +import type { DeviceInfo } from '../../utils/device.ts'; + +type IosSimulatorCameraHelperState = { + pid: number; + shmName: string; + videoPath: string; + logPath: string; + startedAt: string; +}; + +export type IosSimulatorCameraLaunch = { + env: NodeJS.ProcessEnv; + videoPath: string; + shmName: string; + helperPid: number; +}; + +const CAMERA_VENDOR_ROOT = path.join('third_party', 'serve-sim-camera'); +const CAMERA_HELPER_RELATIVE_PATH = path.join( + CAMERA_VENDOR_ROOT, + 'bin', + 'camera-helper', +); +const CAMERA_INJECTOR_RELATIVE_PATH = path.join( + CAMERA_VENDOR_ROOT, + 'bin', + 'camera-injector.dylib', +); + +export async function prepareIosSimulatorCameraVideo(params: { + device: DeviceInfo; + bundleId: string; + videoPath: string; +}): Promise { + assertIosSimulatorCameraSupported(params.device); + const videoPath = await resolveReadableVideoPath(params.videoPath); + const helperPath = resolveVendorExecutable(CAMERA_HELPER_RELATIVE_PATH); + const injectorPath = resolveVendorExecutable(CAMERA_INJECTOR_RELATIVE_PATH); + await stopIosSimulatorCameraVideo(params.device, params.bundleId); + + const shmName = buildShmName(params.device.id, params.bundleId); + const logPath = helperLogPath(params.device, params.bundleId); + await fsp.mkdir(path.dirname(logPath), { recursive: true }); + const logFd = fs.openSync(logPath, 'w'); + let helperPid = 0; + try { + helperPid = runCmdDetached( + helperPath, + ['--shm', shmName, '--source', 'video', '--arg', videoPath], + { + stdio: ['ignore', logFd, logFd], + }, + ); + } finally { + fs.closeSync(logFd); + } + await writeHelperState(params.device, params.bundleId, { + pid: helperPid, + shmName, + videoPath, + logPath, + startedAt: new Date().toISOString(), + }); + + return { + videoPath, + shmName, + helperPid, + env: { + SIMCTL_CHILD_DYLD_INSERT_LIBRARIES: injectorPath, + // Upstream serve-sim injector ABI. simctl strips the SIMCTL_CHILD_ prefix. + SIMCTL_CHILD_SIMCAM_SHM_NAME: shmName, + SIMCTL_CHILD_SIMCAM_MIRROR_MODE: 'auto', + }, + }; +} + +export async function stopIosSimulatorCameraVideo( + device: DeviceInfo, + bundleId: string | undefined, +): Promise { + if (device.platform !== 'ios' || device.kind !== 'simulator' || !bundleId) return; + const statePath = helperStatePath(device, bundleId); + const state = await readHelperState(statePath); + if (!state) return; + try { + if (state.pid > 0) { + process.kill(state.pid, 'SIGTERM'); + } + } catch (error) { + if (!isMissingProcessError(error)) throw error; + } finally { + await fsp.rm(statePath, { force: true }); + } +} + +function assertIosSimulatorCameraSupported(device: DeviceInfo): void { + if (device.platform === 'ios' && device.kind === 'simulator') return; + throw new AppError( + 'UNSUPPORTED_OPERATION', + '--camera-video is supported only for iOS simulators.', + { + platform: device.platform, + kind: device.kind, + }, + ); +} + +async function resolveReadableVideoPath(value: string): Promise { + const resolvedPath = path.resolve(value); + try { + const stat = await fsp.stat(resolvedPath); + if (stat.isFile()) return resolvedPath; + } catch {} + throw new AppError('INVALID_ARGS', `Camera video file does not exist: ${resolvedPath}`, { + hint: 'Pass a readable sample video path to --camera-video.', + }); +} + +function resolveVendorExecutable(relativePath: string): string { + const executablePath = path.join(findProjectRoot(), relativePath); + if (fs.existsSync(executablePath)) return executablePath; + throw new AppError('COMMAND_FAILED', 'Bundled iOS simulator camera helper is missing.', { + expectedPath: executablePath, + }); +} + +function buildShmName(deviceId: string, bundleId: string): string { + const hash = createHash('sha1') + .update(`${deviceId}:${bundleId}:${Date.now()}`) + .digest('hex') + .slice(0, 12); + return `/ad-camera-${hash}`; +} + +function helperStatePath(device: DeviceInfo, bundleId: string): string { + const key = `${device.id}-${bundleId}`.replaceAll(/[^A-Za-z0-9._-]/g, '-'); + return path.join(os.tmpdir(), 'agent-device-ios-camera', `${key}.json`); +} + +function helperLogPath(device: DeviceInfo, bundleId: string): string { + const key = `${device.id}-${bundleId}`.replaceAll(/[^A-Za-z0-9._-]/g, '-'); + return path.join(os.tmpdir(), 'agent-device-ios-camera', `${key}.log`); +} + +async function writeHelperState( + device: DeviceInfo, + bundleId: string, + state: IosSimulatorCameraHelperState, +): Promise { + const statePath = helperStatePath(device, bundleId); + await fsp.mkdir(path.dirname(statePath), { recursive: true }); + await fsp.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8'); +} + +async function readHelperState( + statePath: string, +): Promise { + try { + const state = JSON.parse( + await fsp.readFile(statePath, 'utf8'), + ) as Partial; + if ( + typeof state.pid === 'number' && + typeof state.shmName === 'string' && + typeof state.videoPath === 'string' && + typeof state.logPath === 'string' && + typeof state.startedAt === 'string' + ) { + return state as IosSimulatorCameraHelperState; + } + } catch {} + return undefined; +} + +function isMissingProcessError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: unknown }).code === 'ESRCH' + ); +} diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 6f12b100f..ad55a215b 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -22,6 +22,24 @@ test('parseArgs recognizes command-specific flag combinations', async () => { assert.equal(parsed.flags.relaunch, true); }, }, + { + label: 'open --camera-video', + argv: [ + 'open', + 'com.example.app', + '--platform', + 'ios', + '--camera-video', + './fixtures/back.mp4', + ], + strictFlags: true, + assertParsed: (parsed) => { + assert.equal(parsed.command, 'open'); + assert.deepEqual(parsed.positionals, ['com.example.app']); + assert.equal(parsed.flags.platform, 'ios'); + assert.equal(parsed.flags.cameraVideo, './fixtures/back.mp4'); + }, + }, { label: 'open --platform ios --target tv', argv: ['open', 'Settings', '--platform', 'ios', '--target', 'tv'], @@ -34,13 +52,26 @@ test('parseArgs recognizes command-specific flag combinations', async () => { }, { label: 'boot --headless on android', - argv: ['boot', '--platform', 'android', '--device', 'Pixel_9_Pro_XL', '--headless'], + argv: [ + 'boot', + '--platform', + 'android', + '--device', + 'Pixel_9_Pro_XL', + '--headless', + '--camera-front', + '/tmp/front.mp4', + '--camera-back', + '/tmp/back.mp4', + ], strictFlags: true, assertParsed: (parsed) => { assert.equal(parsed.command, 'boot'); assert.equal(parsed.flags.platform, 'android'); assert.equal(parsed.flags.device, 'Pixel_9_Pro_XL'); assert.equal(parsed.flags.headless, true); + assert.equal(parsed.flags.cameraFront, '/tmp/front.mp4'); + assert.equal(parsed.flags.cameraBack, '/tmp/back.mp4'); }, }, { diff --git a/src/utils/cli-command-overrides.ts b/src/utils/cli-command-overrides.ts index 0b1769526..9f456f557 100644 --- a/src/utils/cli-command-overrides.ts +++ b/src/utils/cli-command-overrides.ts @@ -63,7 +63,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { const CLI_COMMAND_OVERRIDES = { boot: { summary: 'Boot target device/simulator', - allowedFlags: ['headless'], + allowedFlags: ['headless', 'cameraFront', 'cameraBack'], }, shutdown: { summary: 'Shutdown target simulator/emulator', @@ -82,7 +82,15 @@ const CLI_COMMAND_OVERRIDES = { 'Boot device/simulator; optionally launch app or deep link URL (macOS also supports --surface app|frontmost-app|desktop|menubar)', summary: 'Open an app, deep link or URL, save replays', positionalArgs: ['appOrUrl?', 'url?'], - allowedFlags: ['activity', 'launchConsole', 'launchArgs', 'saveScript', 'relaunch', 'surface'], + allowedFlags: [ + 'activity', + 'cameraVideo', + 'launchConsole', + 'launchArgs', + 'saveScript', + 'relaunch', + 'surface', + ], }, close: { positionalArgs: ['app?'], diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index 4a9ce9a37..ec07471d9 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -80,6 +80,7 @@ export type CliFlags = RemoteConfigMetroOptions & pauseMs?: number; pattern?: SwipePattern; activity?: string; + cameraVideo?: string; launchConsole?: string; launchArgs?: string[]; header?: string[]; @@ -90,6 +91,8 @@ export type CliFlags = RemoteConfigMetroOptions & relaunch?: boolean; surface?: SessionSurface; headless?: boolean; + cameraFront?: string; + cameraBack?: string; restart?: boolean; noRecord?: boolean; retainPaths?: boolean; @@ -359,6 +362,22 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--headless', usageDescription: 'Boot: launch Android emulator without a GUI window', }, + { + key: 'cameraFront', + names: ['--camera-front'], + type: 'string', + usageLabel: '--camera-front ', + usageDescription: + 'Boot: Android emulator front camera mode or video file path (for example emulated, none, webcam0, or ./front.mp4)', + }, + { + key: 'cameraBack', + names: ['--camera-back'], + type: 'string', + usageLabel: '--camera-back ', + usageDescription: + 'Boot: Android emulator back camera mode or video file path (for example virtualscene, emulated, none, webcam0, or ./back.mp4)', + }, { key: 'metroHost', names: ['--metro-host'], @@ -514,6 +533,13 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--launch-console ', usageDescription: 'open: capture the initial iOS simulator launch console window to a file', }, + { + key: 'cameraVideo', + names: ['--camera-video'], + type: 'string', + usageLabel: '--camera-video ', + usageDescription: 'open: iOS simulator video file injected as the app camera stream', + }, { key: 'launchArgs', names: ['--launch-args'], diff --git a/third_party/serve-sim-camera/LICENSE b/third_party/serve-sim-camera/LICENSE new file mode 100644 index 000000000..1d3e4e425 --- /dev/null +++ b/third_party/serve-sim-camera/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Evan Bacon + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/serve-sim-camera/README.md b/third_party/serve-sim-camera/README.md new file mode 100644 index 000000000..072bc3192 --- /dev/null +++ b/third_party/serve-sim-camera/README.md @@ -0,0 +1,20 @@ +# serve-sim camera vendor + +This directory vendors the iOS simulator camera helper and injector from +`serve-sim`. + +- Upstream: https://github.com/EvanBacon/serve-sim +- Imported package: `serve-sim@0.1.34` +- License: Apache-2.0, copied in `LICENSE` +- Imported paths: + - `bin/camera-injector.dylib` + - `bin/camera-helper` + +The imported binaries were renamed locally to avoid exposing upstream internal +artifact names in this codebase. + +Local integration code lives outside this directory. Keep local modifications +to vendored artifacts minimal; when changing copied upstream artifacts, +document the change here and preserve Apache-2.0 attribution. + +Current local modifications: none. diff --git a/third_party/serve-sim-camera/bin/camera-helper b/third_party/serve-sim-camera/bin/camera-helper new file mode 100755 index 000000000..dd373a71a Binary files /dev/null and b/third_party/serve-sim-camera/bin/camera-helper differ diff --git a/third_party/serve-sim-camera/bin/camera-injector.dylib b/third_party/serve-sim-camera/bin/camera-injector.dylib new file mode 100755 index 000000000..000a4b6ae Binary files /dev/null and b/third_party/serve-sim-camera/bin/camera-injector.dylib differ diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index efa594947..5821e49f5 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -120,6 +120,9 @@ await client.sessions.close(); For direct iOS simulator app launches, `client.apps.open({ app, platform: 'ios', launchConsole: './artifacts/app.console.log' })` captures launch-time stdout/stderr. The option mirrors `open --launch-console` and is not valid for URL opens or non-simulator targets. +For iOS simulator camera tests, `client.apps.open({ app, platform: 'ios', cameraVideo: './fixtures/camera-feed.mp4' })` injects the video file as the +target app's camera stream for that launch. It relaunches the app process and is not valid for URL-only opens, physical devices, Android, macOS, or Linux. + ## Android snapshot helper providers Remote Android providers should import `agent-device/android-snapshot-helper` and inject their own @@ -257,6 +260,19 @@ Additional CLI-backed methods are exposed on their domain groups with typed opti - `client.recording.record()` and `client.recording.trace()` - `client.settings.update()` +`client.devices.boot({ platform: 'android', device: 'Pixel_9_Pro_XL', headless: true })` starts an Android emulator without a GUI when it is not already running. To launch with emulator camera inputs, pass `cameraFront` and/or `cameraBack` with `emulated`, `none`, `webcam`, `virtualscene` for the back camera, or a video file path: + +```ts +await client.devices.boot({ + platform: 'android', + device: 'Pixel_9_Pro_XL', + cameraFront: './front.mp4', + cameraBack: 'virtualscene', +}); +``` + +Camera inputs are Android-emulator-only and apply only when starting the emulator; shut down a running emulator before changing them. + `client.observability.perf()` returns daemon-shaped JSON so local and remote transports expose the same metrics payload. Pass `{ area: 'metrics' }` for the broad startup/CPU/memory/frame first pass, or `{ area: 'frames' }` for a focused frame/jank-health payload. On Android and supported Apple targets, `data.metrics.fps.droppedFramePercent` is the primary frame-smoothness value. Android derives it from the current `adb shell dumpsys gfxinfo framestats` window; connected iOS devices derive it from `xcrun xctrace` Animation Hitches for the active app process. Frame samples include `windowStartedAt`, `windowEndedAt`, and `worstWindows` so agents can correlate dropped-frame clusters with logs, network entries, and their own session actions. A successful Android read resets Android frame stats; `open ` resets the Android frame window too, so agents can call `perf({ area: 'frames' })`, perform a transition or gesture, then call it again to inspect that focused window. iOS simulator and macOS app sessions report frame health as unavailable rather than inventing FPS or dropped-frame values. `client.recording.record({ action: 'start', path, quality: 5 })` starts a smaller 50% resolution video; omit `quality` to keep native/current resolution. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 1e59dafa6..b99e927b3 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -38,9 +38,11 @@ agent-device boot agent-device boot --platform ios agent-device boot --platform android agent-device boot --platform android --device Pixel_9_Pro_XL --headless +agent-device boot --platform android --device Pixel_9_Pro_XL --camera-back ./back.mp4 --camera-front none agent-device shutdown --platform ios agent-device shutdown --platform android --device Pixel_9_Pro_XL agent-device open [app|url] [url] +agent-device open com.example.CameraApp --platform ios --camera-video ./camera-feed.mp4 agent-device open --platform macos --surface frontmost-app agent-device open --platform macos --surface desktop agent-device close [app] @@ -62,10 +64,13 @@ agent-device app-switcher - `boot` is mainly needed when starting a new session and `open` fails because no booted simulator/emulator is available. - Android: `boot --platform android --device ` launches that emulator in GUI mode when needed. - Android: add `--headless` to launch without opening a GUI window. +- Android: add `--camera-front ` and/or `--camera-back ` when launching an emulator with camera inputs. Supported modes are `emulated`, `none`, `webcam`, and `virtualscene` for the back camera only. File paths are converted to `videofile:` for the emulator. +- Android camera inputs apply only when starting an emulator. Shut down an already-running emulator first, then run `boot` again with the camera flags. - Android: `shutdown --platform android --device ` stops a running emulator. - `open [app|url] [url]` already boots/activates the selected target when needed. - `open ` deep links are supported on Android and iOS. - `open ` opens a deep link on iOS. +- `open --camera-video ` injects a sample video file as the iOS simulator camera stream for that app launch. It relaunches the target app process and is not valid for URL-only opens, physical devices, Android, macOS, or Linux. - `open --launch-console ` captures launch-time stdout/stderr for direct iOS simulator app launches. It is not valid for URL opens or non-simulator targets. - `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface explicitly. `app` is the default when an app argument is provided.