From 04d471f887c500d34185906d900a26ec0d8b7470 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 8 May 2026 17:09:28 -0400 Subject: [PATCH] Improve H264 streaming fallback --- README.md | 8 +- cli/XCWH264Encoder.m | 67 +- client/src/app/AppShell.tsx | 205 ++- .../src/features/simulators/SimulatorMenu.tsx | 108 +- .../features/simulators/useSimulatorList.ts | 18 +- client/src/features/stream/stats.ts | 4 + client/src/features/stream/streamTypes.ts | 8 + .../src/features/stream/streamWorkerClient.ts | 1312 ++++++++++++++++- client/src/features/stream/useLiveStream.ts | 140 +- client/src/features/toolbar/Toolbar.tsx | 7 + client/src/styles/components.css | 34 + docs/api/health.md | 2 +- docs/api/rest.md | 82 +- docs/cli/flags.md | 22 +- docs/contributing.md | 2 +- docs/guide/architecture.md | 24 +- docs/guide/daemon.md | 22 +- docs/guide/video.md | 56 +- server/src/api/routes.rs | 355 ++++- server/src/main.rs | 27 +- server/src/metrics/counters.rs | 10 + server/src/simulators/session.rs | 1 + server/src/transport/webrtc.rs | 57 +- skills/simdeck/SKILL.md | 3 + 24 files changed, 2277 insertions(+), 297 deletions(-) diff --git a/README.md b/README.md index 4273860f..380626a9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ view inside the editor. ## Features -- Local simulator video stream over browser-native WebRTC H.264 +- Local simulator video stream over browser-native WebRTC H.264 with H.264 WebSocket fallback - Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - CoreSimulator chrome asset rendering for device bezels @@ -68,8 +68,10 @@ LAN clients should pair with the printed code before receiving the API cookie. SimDeck Studio providers run the daemon on loopback and use `scripts/studio-provider-bridge.mjs` for outbound control-plane communication with Studio. Studio hosts the browser UI and proxies SimDeck REST requests over -that bridge while WebRTC media still negotiates directly between the browser and -runner through ICE. +that bridge while WebRTC media negotiates directly between the browser and +runner through ICE. If WebRTC fails before rendering a frame, the browser can +fall back to H.264 over WebSocket while keeping input on a separate WebSocket +channel. Expose a local simulator through SimDeck Studio with one command: diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index eb59be5a..33397ceb 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -36,8 +36,6 @@ static const uint64_t XCWLowLatencySoftwareMaximumFrameIntervalUs = 133333; static const uint64_t XCWLowLatencySoftwareFrameIntervalStepUs = 11111; static const NSUInteger XCWLowLatencySoftwareHealthyFrameWindow = 8; -static const uint64_t XCWRealtimeHardwareFrameIntervalStepUs = 5556; -static const NSUInteger XCWRealtimeHardwareHealthyFrameWindow = 6; static const NSUInteger XCWMaximumRealtimeInFlightFrames = 3; static const int32_t XCWRealtimeKeyFrameIntervalSeconds = 5; static const double XCWEncoderLatencyEWMAAlpha = 0.2; @@ -157,10 +155,6 @@ static uint64_t XCWLocalStreamFrameIntervalUs(void) { return (uint64_t)llround(1000000.0 / (double)fps); } -static uint64_t XCWLocalStreamMaximumFrameIntervalUs(void) { - return MAX(XCWSoftwareMaximumFrameIntervalUs, XCWLocalStreamFrameIntervalUs()); -} - static int64_t XCWRealtimeBitsPerPixelBudgetValue(void) { return XCWInt64FromEnvironment(@"SIMDECK_REALTIME_BITS_PER_PIXEL", XCWRealtimeBitsPerPixelBudget, @@ -539,7 +533,6 @@ @implementation XCWH264Encoder { uint64_t _hardwareFrameIntervalUs; uint64_t _lastHardwareSubmissionUs; NSUInteger _hardwarePacedFrameCount; - NSUInteger _hardwareHealthyFrameCount; NSString *_selectedEncoderID; NSInteger _lastSessionStatus; NSInteger _lastPrepareStatus; @@ -612,6 +605,7 @@ - (void)reconfigureForStreamQualityChange { [self invalidateCompressionSessionLocked]; self->_encoderMode = XCWVideoEncoderModeFromEnvironment(); self->_lowLatencyMode = (self->_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment(); + self->_realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || self->_lowLatencyMode; self->_codecType = XCWVideoCodecTypeForMode(self->_encoderMode); self->_needsKeyFrame = YES; self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; @@ -619,7 +613,6 @@ - (void)reconfigureForStreamQualityChange { self->_softwareHealthyFrameCount = 0; self->_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; self->_hardwarePacedFrameCount = 0; - self->_hardwareHealthyFrameCount = 0; }); } @@ -689,9 +682,6 @@ - (NSDictionary *)statsRepresentation { @"hardwareFrameIntervalUs": @(self->_hardwareFrameIntervalUs), @"hardwareTargetFps": @(self->_hardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_hardwareFrameIntervalUs) : 0.0), @"hardwarePacedFrames": @(self->_hardwarePacedFrameCount), - @"realtimeHardwareFrameIntervalUs": @(self->_hardwareFrameIntervalUs), - @"realtimeHardwareTargetFps": @(self->_hardwareFrameIntervalUs > 0 ? (1000000.0 / (double)self->_hardwareFrameIntervalUs) : 0.0), - @"realtimeHardwarePacedFrames": @(self->_hardwarePacedFrameCount), @"transportCodec": XCWCodecName(self->_codecType), @"encoderMode": XCWVideoEncoderModeName(self->_encoderMode), @"lowLatencyMode": @(self->_lowLatencyMode), @@ -770,14 +760,6 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked { return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWLocalStreamFrameIntervalUs(); } -- (uint64_t)maximumHardwareFrameIntervalUsLocked { - if (_realtimeStreamMode) { - uint64_t minimumFpsIntervalUs = (uint64_t)llround(1000000.0 / (double)XCWMinimumLocalStreamFrameRate); - return MAX(XCWRealtimeMaximumFrameIntervalUs(), minimumFpsIntervalUs); - } - return XCWLocalStreamMaximumFrameIntervalUs(); -} - - (uint64_t)activeFrameIntervalUsLocked { if (_encoderMode == XCWVideoEncoderModeH264Software) { return _softwareFrameIntervalUs > 0 ? _softwareFrameIntervalUs : [self initialSoftwareFrameIntervalUsLocked]; @@ -806,6 +788,9 @@ - (BOOL)shouldPaceHardwareFrameAtTimeUs:(uint64_t)nowUs { if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || _needsKeyFrame) { return NO; } + if (_realtimeStreamMode) { + return NO; + } if (_hardwareFrameIntervalUs == 0) { _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; } @@ -877,43 +862,6 @@ - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { _softwareHealthyFrameCount = 0; } -- (void)adaptHardwarePacingForLatencyUs:(uint64_t)latencyUs { - if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || !_realtimeStreamMode || latencyUs == 0) { - return; - } - if (_hardwareFrameIntervalUs == 0) { - _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; - } - - uint64_t minimumIntervalUs = [self minimumHardwareFrameIntervalUsLocked]; - uint64_t maximumIntervalUs = [self maximumHardwareFrameIntervalUsLocked]; - if (latencyUs > _hardwareFrameIntervalUs) { - uint64_t nextIntervalUs = _hardwareFrameIntervalUs + XCWRealtimeHardwareFrameIntervalStepUs; - uint64_t latencyBoundIntervalUs = latencyUs + XCWRealtimeHardwareFrameIntervalStepUs; - if (nextIntervalUs < latencyBoundIntervalUs) { - nextIntervalUs = latencyBoundIntervalUs; - } - _hardwareFrameIntervalUs = MIN(nextIntervalUs, maximumIntervalUs); - _hardwareHealthyFrameCount = 0; - return; - } - - if (latencyUs < _hardwareFrameIntervalUs && - _hardwareFrameIntervalUs > minimumIntervalUs) { - _hardwareHealthyFrameCount += 1; - if (_hardwareHealthyFrameCount >= XCWRealtimeHardwareHealthyFrameWindow) { - uint64_t nextIntervalUs = _hardwareFrameIntervalUs > XCWRealtimeHardwareFrameIntervalStepUs - ? _hardwareFrameIntervalUs - XCWRealtimeHardwareFrameIntervalStepUs - : minimumIntervalUs; - _hardwareFrameIntervalUs = MAX(nextIntervalUs, minimumIntervalUs); - _hardwareHealthyFrameCount = 0; - } - return; - } - - _hardwareHealthyFrameCount = 0; -} - - (void)drainPendingFramesLocked { while (YES) { if (_inFlightFrameCount >= [self maximumInFlightFrameCountLocked]) { @@ -1022,7 +970,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height if (encoderID.length > 0) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EncoderID] = encoderID; } - if (_encoderMode != XCWVideoEncoderModeH264Software && _lowLatencyMode) { + if (_encoderMode != XCWVideoEncoderModeH264Software && (_lowLatencyMode || _realtimeStreamMode)) { if (@available(macOS 11.3, *)) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = @YES; } @@ -1064,7 +1012,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height if (@available(macOS 10.14, *)) { VTSessionSetProperty(session, kVTCompressionPropertyKey_MaximizePowerEfficiency, kCFBooleanFalse); } - if (_lowLatencyMode) { + if (_lowLatencyMode || _realtimeStreamMode) { XCWApplyCompressionPresetIfAvailable(session); } VTSessionSetProperty(session, kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue); @@ -1099,7 +1047,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height if (@available(macOS 11.0, *)) { VTSessionSetProperty(session, kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, - _lowLatencyMode ? kCFBooleanTrue : kCFBooleanFalse); + (_lowLatencyMode || _realtimeStreamMode) ? kCFBooleanTrue : kCFBooleanFalse); } if (@available(macOS 15.0, *)) { VTSessionSetProperty(session, @@ -1400,7 +1348,6 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer } _wasOverloaded = overloaded; [self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs]; - [self adaptHardwarePacingForLatencyUs:_latestEncodeLatencyUs]; } NSString *codec = nil; NSData *decoderConfig = nil; diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 05bcad6a..c4c4666b 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -54,6 +54,7 @@ import type { StreamEncoder, StreamFps, StreamQualityPreset, + StreamTransport, } from "../features/stream/streamTypes"; import { useLiveStream } from "../features/stream/useLiveStream"; import { DebugPanel } from "../features/toolbar/DebugPanel"; @@ -105,21 +106,29 @@ const LOGICAL_INSPECTOR_MAX_DEPTH = 80; const AUTH_REQUIRED_MESSAGE = "SimDeck API access token is required."; const LOCAL_STREAM_DEFAULTS: StreamConfig = { encoder: "auto", - fps: 120, - quality: "quality", + fps: 60, + quality: "full", }; const REMOTE_STREAM_DEFAULTS: StreamConfig = { encoder: "software", fps: 30, quality: "balanced", }; -const STREAM_CONFIG_SYNC_INTERVAL_MS = 1500; +const H264_WS_DEFAULT_FPS = 60; +const H264_WS_LOCAL_DEFAULT_QUALITY: StreamQualityPreset = "full"; +const H264_WS_REMOTE_DEFAULT_QUALITY: StreamQualityPreset = "auto"; +const CONTROL_BACKLOG_DROP_BYTES = 4096; const STREAM_CONFIG_USER_CHANGE_GRACE_MS = 1000; const STREAM_ENCODER_VALUES = new Set([ "auto", "hardware", "software", ]); +const STREAM_TRANSPORT_VALUES = new Set([ + "auto", + "h264", + "webrtc", +]); clearLegacyVolatileUiState(); interface StreamQualityResponse { @@ -166,6 +175,48 @@ function shouldUseRemoteStreamDefault(apiRoot: string): boolean { ); } +function readStreamTransportQueryParam(): StreamTransport { + const value = new URLSearchParams(window.location.search).get("stream"); + if (value === "h264-ws") { + return "h264"; + } + return value && STREAM_TRANSPORT_VALUES.has(value as StreamTransport) + ? (value as StreamTransport) + : "auto"; +} + +function defaultStreamConfigForTransport( + remote: boolean, + transport: StreamTransport, +): StreamConfig { + const base = remote ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS; + if (transport === "h264") { + return { + ...base, + fps: H264_WS_DEFAULT_FPS, + maxEdge: undefined, + quality: remote + ? H264_WS_REMOTE_DEFAULT_QUALITY + : H264_WS_LOCAL_DEFAULT_QUALITY, + }; + } + return base; +} + +function writeStreamTransportQueryParam(transport: StreamTransport) { + const url = new URL(window.location.href); + if (transport === "auto") { + url.searchParams.delete("stream"); + } else { + url.searchParams.set("stream", transport); + } + window.history.replaceState( + null, + "", + `${url.pathname}${url.search}${url.hash}`, + ); +} + function simulatorDisplaySize( simulator: SimulatorMetadata | null, ): Size | null { @@ -246,6 +297,9 @@ export function AppShell({ remoteStream = shouldUseRemoteStreamDefault(apiRoot), }: AppShellProps = {}) { configureSimDeckClient({ apiRoot }); + const initialStreamTransportRef = useRef( + readStreamTransportQueryParam(), + ); const [initialUiState] = useState(readPersistedUiState); const [initialSelectedUDID] = useState( () => @@ -328,7 +382,13 @@ export function AppShell({ readStoredFlag(TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, true), ); const [streamConfig, setStreamConfig] = useState(() => - remoteStream ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS, + defaultStreamConfigForTransport( + remoteStream, + initialStreamTransportRef.current, + ), + ); + const [streamTransport, setStreamTransport] = useState( + initialStreamTransportRef.current, ); const [streamConfigApplyKey, setStreamConfigApplyKey] = useState(0); const [streamConfigReady, setStreamConfigReady] = useState(false); @@ -351,11 +411,17 @@ export function AppShell({ const accessibilityLoadingRef = useRef(false); const streamConfigRequestIdRef = useRef(0); const streamConfigUserChangeAtRef = useRef(0); + const streamConfigUserTouchedRef = useRef(false); const controlSocketRef = useRef<{ udid: string; socket: WebSocket; pending: string[]; } | null>(null); + const pendingTouchMoveRef = useRef<{ + coords: Point; + udid: string; + } | null>(null); + const touchMoveFrameRef = useRef(0); const canvasSize = useElementSize(outerCanvasElement); const zoomDockSize = useElementSize(zoomDockElement); @@ -448,8 +514,13 @@ export function AppShell({ ) { return; } + if (streamTransport === "h264" && !streamConfigUserTouchedRef.current) { + return; + } setStreamConfig((current) => - mergeStreamQualityResponse(current, response), + mergeStreamQualityResponse(current, response, { + preserveAutoQuality: streamTransport === "h264", + }), ); } catch { // Keep the existing local/default selection; the stream path will surface @@ -459,7 +530,7 @@ export function AppShell({ setStreamConfigReady(true); } } - }, []); + }, [streamTransport]); useEffect(() => { let cancelled = false; @@ -472,10 +543,8 @@ export function AppShell({ }; run(); - const intervalId = window.setInterval(run, STREAM_CONFIG_SYNC_INTERVAL_MS); return () => { cancelled = true; - window.clearInterval(intervalId); }; }, [remoteStream, syncStreamConfig]); @@ -496,9 +565,11 @@ export function AppShell({ simulator: selectedSimulator, streamConfig, streamConfigApplyKey, + streamTransport, }); const updateStreamEncoder = useCallback((encoder: StreamEncoder) => { + streamConfigUserTouchedRef.current = true; streamConfigUserChangeAtRef.current = Date.now(); setStreamConfigReady(true); setStreamConfigApplyKey((current) => current + 1); @@ -506,6 +577,7 @@ export function AppShell({ }, []); const updateStreamFps = useCallback((fps: StreamFps) => { + streamConfigUserTouchedRef.current = true; streamConfigUserChangeAtRef.current = Date.now(); setStreamConfigReady(true); setStreamConfigApplyKey((current) => current + 1); @@ -513,12 +585,35 @@ export function AppShell({ }, []); const updateStreamQuality = useCallback((quality: StreamQualityPreset) => { + streamConfigUserTouchedRef.current = true; streamConfigUserChangeAtRef.current = Date.now(); setStreamConfigReady(true); setStreamConfigApplyKey((current) => current + 1); setStreamConfig((current) => ({ ...current, maxEdge: undefined, quality })); }, []); + const updateStreamTransport = useCallback( + (transport: StreamTransport) => { + setStreamTransport(transport); + writeStreamTransportQueryParam(transport); + if (transport !== "h264" || streamConfigUserTouchedRef.current) { + return; + } + streamConfigUserChangeAtRef.current = Date.now(); + setStreamConfigReady(true); + setStreamConfigApplyKey((current) => current + 1); + setStreamConfig((current) => ({ + ...current, + fps: H264_WS_DEFAULT_FPS, + maxEdge: undefined, + quality: remoteStream + ? H264_WS_REMOTE_DEFAULT_QUALITY + : H264_WS_LOCAL_DEFAULT_QUALITY, + })); + }, + [remoteStream], + ); + useEffect(() => { if ( !selectedSimulator || @@ -1029,7 +1124,7 @@ export function AppShell({ setAccessibilitySelectedId(""); setAccessibilityHoveredId(null); } - sendControl(selectedSimulator.udid, { type: "touch", ...coords, phase }); + sendTouchControl(selectedSimulator.udid, phase, coords); }, onTouchPreview: showTouchIndicator, pan, @@ -1204,9 +1299,52 @@ export function AppShell({ }, []); function sendControl(udid: string, message: ControlMessage): boolean { + if (message.type === "touch") { + sendTouchControl(udid, message.phase, { x: message.x, y: message.y }); + return true; + } + return sendControlNow(udid, message); + } + + function sendTouchControl(udid: string, phase: TouchPhase, coords: Point) { + if (phase === "moved") { + pendingTouchMoveRef.current = { coords, udid }; + if (!touchMoveFrameRef.current) { + touchMoveFrameRef.current = window.requestAnimationFrame(() => { + touchMoveFrameRef.current = 0; + flushPendingTouchMove(); + }); + } + return; + } + + flushPendingTouchMove(); + sendControlNow(udid, { type: "touch", ...coords, phase }); + } + + function flushPendingTouchMove() { + const pending = pendingTouchMoveRef.current; + pendingTouchMoveRef.current = null; + if (touchMoveFrameRef.current) { + window.cancelAnimationFrame(touchMoveFrameRef.current); + touchMoveFrameRef.current = 0; + } + if (!pending) { + return; + } + sendControlNow(pending.udid, { + type: "touch", + ...pending.coords, + phase: "moved", + }); + } + + function sendControlNow(udid: string, message: ControlMessage): boolean { setLocalError(""); const encoded = JSON.stringify(message); - if (sendWebRtcControlMessage(encoded)) { + const dropIfBacklogged = + message.type === "touch" && message.phase === "moved"; + if (sendWebRtcControlMessage(encoded, { dropIfBacklogged })) { return true; } if (remoteStream) { @@ -1214,14 +1352,36 @@ export function AppShell({ } const state = ensureControlSocket(udid); if (state.socket.readyState === WebSocket.OPEN) { + if ( + dropIfBacklogged && + state.socket.bufferedAmount > CONTROL_BACKLOG_DROP_BYTES + ) { + return true; + } state.socket.send(encoded); } else { + if (dropIfBacklogged) { + const lastIndex = state.pending.length - 1; + if (lastIndex >= 0 && state.pending[lastIndex].includes('"moved"')) { + state.pending[lastIndex] = encoded; + return true; + } + } state.pending.push(encoded); } return true; } - useEffect(() => closeControlSocket, [closeControlSocket]); + useEffect(() => { + return () => { + if (touchMoveFrameRef.current) { + window.cancelAnimationFrame(touchMoveFrameRef.current); + touchMoveFrameRef.current = 0; + } + pendingTouchMoveRef.current = null; + closeControlSocket(); + }; + }, [closeControlSocket]); function beginZoomAnimation() { setZoomAnimating(true); @@ -1574,6 +1734,7 @@ export function AppShell({ onStreamEncoderChange={updateStreamEncoder} onStreamFpsChange={updateStreamFps} onStreamQualityChange={updateStreamQuality} + onStreamTransportChange={updateStreamTransport} onShutdown={() => { if (!selectedSimulator) { return; @@ -1622,6 +1783,7 @@ export function AppShell({ !selectedSimulatorTransitionKind, )} streamConfig={streamConfig} + streamTransport={streamTransport} showStopButton={Boolean( selectedSimulator?.isBooted && !selectedSimulatorTransitionKind, )} @@ -1779,6 +1941,7 @@ function friendlyStreamError( function mergeStreamQualityResponse( current: StreamConfig, response: StreamQualityResponse, + options: { preserveAutoQuality?: boolean } = {}, ): StreamConfig { const quality = response.quality ?? {}; const next: StreamConfig = { @@ -1789,7 +1952,10 @@ function mergeStreamQualityResponse( ), fps: normalizeStreamFps(quality.fps, current.fps), maxEdge: normalizeMaxEdge(quality.maxEdge, current.maxEdge), - quality: normalizeStreamQuality(quality.profile, current.quality), + quality: + options.preserveAutoQuality && current.quality === "auto" + ? "auto" + : normalizeStreamQuality(quality.profile, current.quality), }; return streamConfigsEqual(current, next) ? current : next; } @@ -1809,14 +1975,19 @@ function normalizeStreamQuality( fallback: StreamQualityPreset, ): StreamQualityPreset { const normalized = value?.trim().toLowerCase(); + if (normalized === "auto") { + return "auto"; + } + if (normalized === "full") { + return "full"; + } if (normalized === "quality") { return "quality"; } - if ( - normalized === "balanced" || - normalized === "smooth" || - normalized === "fast" - ) { + if (normalized === "smooth") { + return "smooth"; + } + if (normalized === "balanced" || normalized === "fast") { return "balanced"; } if (normalized === "economy" || normalized === "ci-software") { diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index 174ccbc1..5c6f0fb3 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -6,6 +6,7 @@ import type { StreamEncoder, StreamFps, StreamQualityPreset, + StreamTransport, } from "../stream/streamTypes"; import { SimulatorRow } from "./SimulatorRow"; @@ -25,6 +26,7 @@ interface SimulatorMenuProps { onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; onStreamQualityChange: (quality: StreamQualityPreset) => void; + onStreamTransportChange: (transport: StreamTransport) => void; onToggleAppearance: () => void; onToggleDebug: () => void; onToggleMenu: () => void; @@ -34,6 +36,7 @@ interface SimulatorMenuProps { selectedSimulator: SimulatorMetadata | null; setSelectedUDID: (udid: string) => void; streamConfig: StreamConfig; + streamTransport: StreamTransport; touchOverlayVisible: boolean; } @@ -53,6 +56,7 @@ export function SimulatorMenu({ onStreamEncoderChange, onStreamFpsChange, onStreamQualityChange, + onStreamTransportChange, onToggleAppearance, onToggleDebug, onToggleMenu, @@ -62,11 +66,23 @@ export function SimulatorMenu({ selectedSimulator, setSelectedUDID, streamConfig, + streamTransport, touchOverlayVisible, }: SimulatorMenuProps) { const fpsOptions = remoteStream ? REMOTE_STREAM_FPS_OPTIONS : LOCAL_STREAM_FPS_OPTIONS; + const qualityOptions = H264_STREAM_QUALITY_OPTIONS; + const activeQualityOption = qualityOptions.some( + (option) => option.value === streamConfig.quality, + ) + ? [] + : [ + { + label: streamQualityOptionLabel(streamConfig.quality), + value: streamConfig.quality, + }, + ]; const activeFpsOption = fpsOptions.some( (option) => option.value === streamConfig.fps, ) @@ -123,9 +139,27 @@ export function SimulatorMenu({
Stream - {formatStreamConfigSummary(streamConfig)} + {formatStreamConfigSummary(streamConfig, streamTransport)}
+
{STREAM_ENCODERS.map((option) => (
-
- {STREAM_QUALITY_OPTIONS.map((option) => ( - - ))} -
+
@@ -221,6 +263,12 @@ const STREAM_ENCODERS: Array<{ label: string; value: StreamEncoder }> = [ { label: "Software", value: "software" }, ]; +const STREAM_TRANSPORTS: Array<{ label: string; value: StreamTransport }> = [ + { label: "Auto", value: "auto" }, + { label: "WebRTC", value: "webrtc" }, + { label: "H264 WS", value: "h264" }, +]; + const LOCAL_STREAM_FPS_OPTIONS: Array<{ label: string; value: StreamFps }> = [ { label: "30", value: 30 }, { label: "60", value: 60 }, @@ -233,17 +281,33 @@ const REMOTE_STREAM_FPS_OPTIONS: Array<{ label: string; value: StreamFps }> = [ { label: "60", value: 60 }, ]; -const STREAM_QUALITY_OPTIONS: Array<{ +const H264_STREAM_QUALITY_OPTIONS: Array<{ label: string; value: StreamQualityPreset; }> = [ - { label: "Full", value: "quality" }, + { label: "Auto", value: "auto" }, + { label: "Full", value: "full" }, { label: "1280", value: "balanced" }, { label: "1080", value: "economy" }, { label: "720", value: "low" }, { label: "540", value: "tiny" }, ]; +const H264_QUALITY_LABELS: Partial> = { + auto: "Auto", + balanced: "1280px", + economy: "1080px", + full: "Full res", + low: "720px", + quality: "Full+", + smooth: "1170px", + tiny: "540px", +}; + +function streamQualityOptionLabel(quality: StreamQualityPreset): string { + return H264_QUALITY_LABELS[quality] ?? quality; +} + function MenuIcon() { return ( @@ -252,10 +316,16 @@ function MenuIcon() { ); } -function formatStreamConfigSummary(streamConfig: StreamConfig): string { +function formatStreamConfigSummary( + streamConfig: StreamConfig, + transport: StreamTransport, +): string { + const transportLabel = + transport === "h264" ? "H264 WS" : transport.toUpperCase(); const resolution = - typeof streamConfig.maxEdge === "number" && streamConfig.maxEdge > 0 + H264_QUALITY_LABELS[streamConfig.quality] ?? + (typeof streamConfig.maxEdge === "number" && streamConfig.maxEdge > 0 ? `${streamConfig.maxEdge}px` - : "Full res"; - return `${resolution} / ${streamConfig.fps} fps`; + : "Full res"); + return `${transportLabel} / ${resolution} / ${streamConfig.fps} fps`; } diff --git a/client/src/features/simulators/useSimulatorList.ts b/client/src/features/simulators/useSimulatorList.ts index a9492742..1fb8abfb 100644 --- a/client/src/features/simulators/useSimulatorList.ts +++ b/client/src/features/simulators/useSimulatorList.ts @@ -3,8 +3,8 @@ import { startTransition, useEffect, useRef, useState } from "react"; import { listSimulators } from "../../api/simulators"; import type { SimulatorMetadata } from "../../api/types"; -const LOCAL_REFRESH_MS = 5000; -const REMOTE_REFRESH_MS = 10000; +const LOCAL_REFRESH_MS = 30000; +const REMOTE_REFRESH_MS = 60000; const REMOTE_ERROR_REFRESH_MS = 15000; const REMOTE_REQUEST_TIMEOUT_MS = 12000; @@ -89,13 +89,27 @@ export function useSimulatorList({ void loadSimulators(cancelled).finally(scheduleNext); }; + const refreshWhenVisible = () => { + if (!cancelled && document.visibilityState === "visible") { + if (timeoutId) { + window.clearTimeout(timeoutId); + timeoutId = 0; + } + run(); + } + }; + run(); + document.addEventListener("visibilitychange", refreshWhenVisible); + window.addEventListener("focus", refreshWhenVisible); return () => { cancelled = true; if (timeoutId) { window.clearTimeout(timeoutId); } + document.removeEventListener("visibilitychange", refreshWhenVisible); + window.removeEventListener("focus", refreshWhenVisible); }; }, [remote]); diff --git a/client/src/features/stream/stats.ts b/client/src/features/stream/stats.ts index 92fd86ad..4162e69f 100644 --- a/client/src/features/stream/stats.ts +++ b/client/src/features/stream/stats.ts @@ -9,6 +9,10 @@ export function createEmptyStreamStats(): StreamStats { decoderDroppedFrames: 0, droppedFrames: 0, frameSequence: 0, + h264ParseFailures: 0, + h264SocketBytes: 0, + h264SocketMessages: 0, + h264SocketMessageType: "", height: 0, iceRestartReason: "", iceRestarts: 0, diff --git a/client/src/features/stream/streamTypes.ts b/client/src/features/stream/streamTypes.ts index 37404a30..f55eb58c 100644 --- a/client/src/features/stream/streamTypes.ts +++ b/client/src/features/stream/streamTypes.ts @@ -4,16 +4,20 @@ export interface StreamConnectTarget { clientId?: string; remote?: boolean; streamConfig?: StreamConfig; + transport?: StreamTransport; udid: string; } export type StreamEncoder = "auto" | "hardware" | "software"; export type StreamFps = number; +export type StreamTransport = "auto" | "h264" | "webrtc"; export type StreamQualityPreset = + | "auto" | "balanced" | "ci-software" | "economy" | "fast" + | "full" | "low" | "quality" | "smooth" @@ -47,6 +51,10 @@ export interface StreamStats extends Size { decoderDroppedFrames: number; droppedFrames: number; frameSequence: number; + h264ParseFailures: number; + h264SocketBytes: number; + h264SocketMessages: number; + h264SocketMessageType: string; iceRestartReason: string; iceRestarts: number; latestFrameGapMs: number; diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 6c0f4b23..e5397753 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -1,11 +1,17 @@ -import { apiHeaders, fetchHealth } from "../../api/client"; +import { + accessTokenFromLocation, + apiHeaders, + fetchHealth, +} from "../../api/client"; import { apiUrl } from "../../api/config"; import type { HealthResponse } from "../../api/types"; import { createEmptyStreamStats } from "./stats"; import type { StreamConnectTarget, StreamConfig, + StreamQualityPreset, StreamStats, + StreamTransport, WorkerToMainMessage, } from "./streamTypes"; @@ -21,21 +27,56 @@ const WEBRTC_REMOTE_DISCONNECTED_GRACE_MS = 10000; const WEBRTC_REMOTE_ICE_RESTART_GRACE_MS = 1500; const WEBRTC_RECONNECT_BASE_DELAY_MS = 250; const WEBRTC_RECONNECT_MAX_DELAY_MS = 1000; +const H264_WS_FIRST_FRAME_TIMEOUT_MS = 10000; +const H264_WS_STALLED_FRAME_TIMEOUT_MS = 5000; +const H264_WS_HEADER_BYTES = 40; +const H264_WS_MAGIC = 0x53444831; +const H264_WS_FLAG_KEYFRAME = 1 << 0; +const H264_WS_FLAG_CONFIG = 1 << 1; +const H264_WS_LOCAL_AUTO_PROFILES: StreamQualityPreset[] = [ + "low", + "economy", + "smooth", + "balanced", + "full", +]; +const H264_WS_REMOTE_AUTO_PROFILES: StreamQualityPreset[] = [ + "tiny", + "low", + "economy", + "smooth", + "balanced", + "full", +]; +const H264_WS_LOCAL_INITIAL_AUTO_PROFILE: StreamQualityPreset = "full"; +const H264_WS_REMOTE_INITIAL_AUTO_PROFILE: StreamQualityPreset = "low"; +const H264_WS_ADAPTIVE_SAMPLE_MS = 1000; +const H264_WS_AUTO_STABLE_SAMPLES_TO_UPGRADE = 3; +const CONTROL_BACKLOG_DROP_BYTES = 4096; let activeWebRtcControlChannel: RTCDataChannel | null = null; let activeWebRtcTelemetryChannel: RTCDataChannel | null = null; +let activeInputSocket: WebSocket | null = null; +let activeH264StreamSocket: WebSocket | null = null; let activeStreamClient: StreamWorkerClient | null = null; -export type StreamBackend = "webrtc"; +export type StreamBackend = "h264-ws" | "webrtc"; -export function sendWebRtcControlMessage(encoded: string): boolean { - return sendDataChannelMessage(activeWebRtcControlChannel, encoded); +export function sendWebRtcControlMessage( + encoded: string, + options: { dropIfBacklogged?: boolean } = {}, +): boolean { + return ( + sendDataChannelMessage(activeWebRtcControlChannel, encoded, options) || + sendWebSocketMessage(activeInputSocket, encoded, options) + ); } -export function sendWebRtcClientStats(stats: unknown): boolean { - return sendDataChannelMessage( - activeWebRtcTelemetryChannel, - JSON.stringify({ stats, type: "clientStats" }), +export function sendStreamClientStats(stats: unknown): boolean { + const encoded = JSON.stringify({ stats, type: "clientStats" }); + return ( + sendDataChannelMessage(activeWebRtcTelemetryChannel, encoded) || + sendWebSocketMessage(activeH264StreamSocket, encoded) ); } @@ -49,17 +90,53 @@ export function sendWebRtcStreamControl(options: { ); } +function sendStreamQualityConfig(config: StreamConfig): boolean { + const encoded = JSON.stringify({ + config: streamQualityPayload(config), + type: "streamQuality", + }); + return ( + sendDataChannelMessage(activeWebRtcControlChannel, encoded) || + sendWebSocketMessage(activeH264StreamSocket, encoded) + ); +} + function sendDataChannelMessage( channel: RTCDataChannel | null, encoded: string, + options: { dropIfBacklogged?: boolean } = {}, ): boolean { if (channel?.readyState !== "open") { return false; } + if ( + options.dropIfBacklogged && + channel.bufferedAmount > CONTROL_BACKLOG_DROP_BYTES + ) { + return true; + } channel.send(encoded); return true; } +function sendWebSocketMessage( + socket: WebSocket | null, + encoded: string, + options: { dropIfBacklogged?: boolean } = {}, +): boolean { + if (socket?.readyState !== WebSocket.OPEN) { + return false; + } + if ( + options.dropIfBacklogged && + socket.bufferedAmount > CONTROL_BACKLOG_DROP_BYTES + ) { + return true; + } + socket.send(encoded); + return true; +} + function compareVideoToImage( video: HTMLVideoElement, source: ImageBitmap, @@ -149,20 +226,50 @@ export function buildStreamTarget( clientId?: string; remote?: boolean; streamConfig?: StreamConfig; + transport?: StreamTransport; } = {}, ): StreamConnectTarget { return { clientId: options.clientId, remote: options.remote, streamConfig: options.streamConfig, + transport: options.transport, udid, }; } +function webSocketApiUrl(path: string): string { + const url = new URL(apiUrl(path), window.location.href); + const token = accessTokenFromLocation(); + if (token) { + url.searchParams.set("simdeckToken", token); + } + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return url.toString(); +} + +function isLoopbackHost(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname.endsWith(".localhost") + ); +} + export function canUseWebRtc(): boolean { return typeof RTCPeerConnection === "function"; } +export function canUseH264WebSocket(): boolean { + const { EncodedVideoChunk, VideoDecoder } = webCodecsConstructors(); + return ( + typeof WebSocket === "function" && + typeof VideoDecoder === "function" && + typeof EncodedVideoChunk === "function" + ); +} + interface StreamClientBackend { attachCanvas(canvasElement: HTMLCanvasElement): void; clear(): void; @@ -183,6 +290,909 @@ export interface VisualArtifactSample { meanDiff: number; } +interface H264WebSocketFrame { + config: Uint8Array; + height: number; + isKeyFrame: boolean; + payload: Uint8Array; + sequence: number; + timestampUs: number; + width: number; +} + +interface WebCodecsVideoFrame { + close(): void; + codedHeight?: number; + codedWidth?: number; + displayHeight?: number; + displayWidth?: number; + timestamp?: number; +} + +interface WebCodecsVideoDecoderConfig { + codec: string; + codedHeight?: number; + codedWidth?: number; + description?: BufferSource; + hardwareAcceleration?: + | "no-preference" + | "prefer-hardware" + | "prefer-software"; + optimizeForLatency?: boolean; +} + +interface WebCodecsVideoDecoder { + readonly decodeQueueSize: number; + readonly state?: string; + close(): void; + configure(config: WebCodecsVideoDecoderConfig): void; + decode(chunk: WebCodecsEncodedVideoChunk): void; +} + +interface WebCodecsEncodedVideoChunk { + readonly byteLength?: number; +} + +interface WebCodecsEncodedVideoChunkConstructor { + new (init: { + data: BufferSource; + timestamp: number; + type: "delta" | "key"; + }): WebCodecsEncodedVideoChunk; +} + +interface WebCodecsVideoDecoderConstructor { + new (init: { + error: (error: Error) => void; + output: (frame: WebCodecsVideoFrame) => void; + }): WebCodecsVideoDecoder; + isConfigSupported?: (config: WebCodecsVideoDecoderConfig) => Promise<{ + supported: boolean; + }>; +} + +interface PendingVideoFrame { + frame: WebCodecsVideoFrame; + sequence: number | null; +} + +function webCodecsConstructors(): { + EncodedVideoChunk?: WebCodecsEncodedVideoChunkConstructor; + VideoDecoder?: WebCodecsVideoDecoderConstructor; +} { + return globalThis as typeof globalThis & { + EncodedVideoChunk?: WebCodecsEncodedVideoChunkConstructor; + VideoDecoder?: WebCodecsVideoDecoderConstructor; + }; +} + +function hasArrayBufferMethod( + value: unknown, +): value is { arrayBuffer: () => Promise } { + return ( + typeof value === "object" && + value !== null && + "arrayBuffer" in value && + typeof (value as { arrayBuffer?: unknown }).arrayBuffer === "function" + ); +} + +class H264WebSocketStreamClient implements StreamClientBackend { + private adaptiveInterval = 0; + private adaptiveLastAt = 0; + private adaptiveLastDecodedFrames = 0; + private adaptiveLastRenderedFrames = 0; + private autoProfile: StreamQualityPreset = + H264_WS_REMOTE_INITIAL_AUTO_PROFILE; + private autoProfileStableSamples = 0; + private animationFrame = 0; + private canvas: HTMLCanvasElement | null = null; + private canvasContext: CanvasRenderingContext2D | null = null; + private connectGeneration = 0; + private decoder: WebCodecsVideoDecoder | null = null; + private decoderConfigKey = ""; + private frameWatchdogTimeout = 0; + private inputSocket: WebSocket | null = null; + private h264DecodeTimestampUs = 0; + private lastFrameAt = 0; + private pendingFrame: PendingVideoFrame | null = null; + private pendingFrameSequences = new Map(); + private reportedVideoHeight = 0; + private reportedVideoWidth = 0; + private shouldReconnect = false; + private stats: StreamStats = createEmptyStreamStats(); + private stalledFrameWatchdogCount = 0; + private streamingReported = false; + private streamConfig?: StreamConfig; + private streamSocket: WebSocket | null = null; + private streamTarget: StreamConnectTarget | null = null; + private waitingForKeyFrame = true; + + constructor( + private readonly onMessage: (message: WorkerToMainMessage) => void, + ) {} + + attachCanvas(canvasElement: HTMLCanvasElement) { + this.canvas = canvasElement; + this.canvasContext = canvasElement.getContext("2d", { + alpha: false, + desynchronized: true, + }); + } + + async connect(target: StreamConnectTarget) { + this.disconnect(); + if (!this.canvas) { + return; + } + if (!canUseH264WebSocket()) { + this.handleError("H264 WebSocket requires browser WebCodecs support."); + return; + } + + const generation = ++this.connectGeneration; + this.autoProfile = initialH264AutoProfile(target); + this.autoProfileStableSamples = 0; + const effectiveConfig = h264WebSocketStreamConfig( + target.streamConfig, + this.autoProfile, + ); + this.shouldReconnect = true; + this.streamTarget = target; + this.streamConfig = target.streamConfig; + this.streamingReported = false; + this.waitingForKeyFrame = true; + this.h264DecodeTimestampUs = 0; + this.stats = createEmptyStreamStats(); + this.stats.codec = "h264-ws"; + this.stats.waitingForKeyFrame = true; + this.lastFrameAt = 0; + this.stalledFrameWatchdogCount = 0; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + this.onMessage({ + type: "status", + status: { detail: "Opening H264 WebSocket stream", state: "connecting" }, + }); + + const socket = new WebSocket( + webSocketApiUrl( + `/api/simulators/${encodeURIComponent(target.udid)}/h264${streamQualityQuery(effectiveConfig)}`, + ), + ); + socket.binaryType = "arraybuffer"; + this.streamSocket = socket; + socket.addEventListener("open", () => { + if (socket === this.streamSocket) { + socket.binaryType = "arraybuffer"; + activeH264StreamSocket = socket; + this.startAdaptiveQuality(generation); + } + }); + socket.addEventListener("message", (event) => { + if (socket !== this.streamSocket) { + return; + } + this.recordH264SocketMessage(event.data); + if (hasArrayBufferMethod(event.data)) { + void event.data.arrayBuffer().then((buffer) => { + if (socket === this.streamSocket) { + this.handleSocketMessage(buffer); + } + }); + return; + } + this.handleSocketMessage(event.data); + }); + socket.addEventListener("close", () => { + if (activeH264StreamSocket === socket) { + activeH264StreamSocket = null; + } + if (socket === this.streamSocket && this.shouldReconnect) { + this.handleError("H264 WebSocket stream closed."); + } + }); + socket.addEventListener("error", () => { + if (socket === this.streamSocket) { + this.handleError("H264 WebSocket stream failed."); + } + }); + + this.connectInputSocket(target, generation); + this.scheduleFrameWatchdog(generation); + } + + disconnect() { + this.shouldReconnect = false; + this.connectGeneration += 1; + this.clearAdaptiveQuality(); + this.clearFrameWatchdog(); + window.cancelAnimationFrame(this.animationFrame); + this.animationFrame = 0; + this.closePendingFrame(); + this.closeDecoder(); + this.pendingFrameSequences.clear(); + this.h264DecodeTimestampUs = 0; + this.streamSocket?.close(); + if (activeH264StreamSocket === this.streamSocket) { + activeH264StreamSocket = null; + } + this.streamSocket = null; + this.inputSocket?.close(); + if (activeInputSocket === this.inputSocket) { + activeInputSocket = null; + } + this.inputSocket = null; + this.streamingReported = false; + this.streamTarget = null; + this.lastFrameAt = 0; + this.stalledFrameWatchdogCount = 0; + this.waitingForKeyFrame = true; + this.reportedVideoHeight = 0; + this.reportedVideoWidth = 0; + } + + destroy() { + this.disconnect(); + } + + clear() { + if (!this.canvas) { + return; + } + this.ensureCanvasContext()?.clearRect( + 0, + 0, + this.canvas.width, + this.canvas.height, + ); + } + + sendControl(payload: unknown): boolean { + if ( + payload && + typeof payload === "object" && + "type" in payload && + payload.type === "streamControl" + ) { + return sendWebSocketMessage(this.streamSocket, JSON.stringify(payload)); + } + return sendWebSocketMessage(this.inputSocket, JSON.stringify(payload)); + } + + async applyStreamConfig(config?: StreamConfig) { + this.autoProfile = initialH264AutoProfile(this.streamTarget); + this.autoProfileStableSamples = 0; + const effectiveConfig = h264WebSocketStreamConfig(config, this.autoProfile); + this.streamConfig = config; + this.clearAdaptiveQuality(); + if (config?.quality === "auto") { + this.startAdaptiveQuality(this.connectGeneration); + } + if (effectiveConfig) { + if (!sendStreamQualityConfig(effectiveConfig)) { + await postStreamConfigWithAuthRetry(effectiveConfig, { + remote: this.streamTarget?.remote, + }); + } + this.sendControl({ + forceKeyframe: true, + snapshot: true, + type: "streamControl", + }); + } + } + + private connectInputSocket(target: StreamConnectTarget, generation: number) { + const socket = new WebSocket( + webSocketApiUrl( + `/api/simulators/${encodeURIComponent(target.udid)}/input`, + ), + ); + this.inputSocket = socket; + activeInputSocket = socket; + socket.addEventListener("open", () => { + if (generation === this.connectGeneration) { + activeInputSocket = socket; + } + }); + socket.addEventListener("close", () => { + if (activeInputSocket === socket) { + activeInputSocket = null; + } + }); + socket.addEventListener("error", () => { + if (generation === this.connectGeneration) { + console.warn("H264 input WebSocket failed."); + } + }); + } + + private handleSocketMessage(data: unknown) { + const frame = parseH264WebSocketFrame(data); + if (!frame) { + this.stats.h264ParseFailures += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + this.stats.receivedPackets += 1; + this.stats.frameSequence = frame.sequence; + this.stats.width = frame.width; + this.stats.height = frame.height; + this.stats.codec = "h264-ws"; + + if (this.waitingForKeyFrame && !frame.isKeyFrame) { + this.stats.droppedFrames += 1; + this.stats.waitingForKeyFrame = true; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + this.sendControl({ forceKeyframe: true, type: "streamControl" }); + return; + } + + if (!this.ensureDecoderConfigured(frame)) { + return; + } + const decoder = this.decoder; + if (!decoder) { + return; + } + if (decoder.state === "closed") { + this.closeDecoder(); + this.waitingForKeyFrame = true; + this.stats.waitingForKeyFrame = true; + this.sendControl({ forceKeyframe: true, type: "streamControl" }); + return; + } + this.stats.decodeQueueSize = decoder.decodeQueueSize; + + const { EncodedVideoChunk } = webCodecsConstructors(); + if (!EncodedVideoChunk) { + this.handleError("H264 WebSocket requires EncodedVideoChunk support."); + return; + } + const decodeTimestampUs = this.nextDecodeTimestampUs(frame.timestampUs); + try { + this.pendingFrameSequences.set(decodeTimestampUs, frame.sequence); + this.trimPendingFrameSequenceMap(); + decoder.decode( + new EncodedVideoChunk({ + data: frame.payload as BufferSource, + timestamp: decodeTimestampUs, + type: frame.isKeyFrame ? "key" : "delta", + }), + ); + this.waitingForKeyFrame = false; + this.stats.waitingForKeyFrame = false; + this.stats.decodeQueueSize = decoder.decodeQueueSize; + } catch (error) { + this.pendingFrameSequences.delete(decodeTimestampUs); + this.closeDecoder(); + this.waitingForKeyFrame = true; + this.stats.waitingForKeyFrame = true; + this.stats.decoderDroppedFrames += 1; + this.sendControl({ forceKeyframe: true, type: "streamControl" }); + this.onMessage({ type: "stats", stats: { ...this.stats } }); + if (!this.streamingReported) { + this.onMessage({ + type: "status", + status: { detail: "Recovering H264 decoder", state: "connecting" }, + }); + } + } + } + + private ensureDecoderConfigured(frame: H264WebSocketFrame): boolean { + const { VideoDecoder } = webCodecsConstructors(); + if (!VideoDecoder) { + this.handleError("H264 WebSocket requires VideoDecoder support."); + return false; + } + if (this.decoder?.state === "closed") { + this.closeDecoder(); + } + if (this.decoder && frame.config.byteLength === 0) { + return true; + } + const configKey = h264DecoderConfigKey(frame); + if (this.decoder && this.decoderConfigKey === configKey) { + return true; + } + + this.closeDecoder(); + const decoder = new VideoDecoder({ + error: (error) => { + this.stats.decoderDroppedFrames += 1; + this.waitingForKeyFrame = true; + this.stats.waitingForKeyFrame = true; + if (this.decoder === decoder) { + this.closeDecoder(); + } + this.sendControl({ forceKeyframe: true, type: "streamControl" }); + this.onMessage({ type: "stats", stats: { ...this.stats } }); + if (!this.streamingReported) { + this.onMessage({ + type: "status", + status: { detail: "Recovering H264 decoder", state: "connecting" }, + }); + } + }, + output: (videoFrame) => this.queueDecodedFrame(videoFrame), + }); + const config: WebCodecsVideoDecoderConfig = { + codec: h264CodecStringFromAvcC(frame.config) ?? "avc1.42E01F", + codedHeight: frame.height || undefined, + codedWidth: frame.width || undefined, + hardwareAcceleration: "prefer-hardware", + optimizeForLatency: true, + }; + if (frame.config.byteLength > 0) { + config.description = arrayBufferCopy(frame.config); + } + try { + decoder.configure(config); + } catch (error) { + decoder.close(); + this.handleError(error instanceof Error ? error.message : String(error)); + return false; + } + this.decoder = decoder; + this.decoderConfigKey = configKey; + return true; + } + + private nextDecodeTimestampUs(sourceTimestampUs: number): number { + const sourceTimestamp = Number.isFinite(sourceTimestampUs) + ? Math.max(0, Math.floor(sourceTimestampUs)) + : 0; + this.h264DecodeTimestampUs = Math.max( + sourceTimestamp, + this.h264DecodeTimestampUs + 1, + ); + return this.h264DecodeTimestampUs; + } + + private queueDecodedFrame(videoFrame: WebCodecsVideoFrame) { + const sequence = this.takeRenderedFrameSequence(videoFrame); + this.stats.decodedFrames += 1; + if (this.pendingFrame) { + this.stats.droppedFrames += 1; + this.closePendingFrame(); + } + this.pendingFrame = { frame: videoFrame, sequence }; + this.schedulePaint(); + } + + private schedulePaint() { + if (this.animationFrame) { + return; + } + this.animationFrame = window.requestAnimationFrame(this.paintPendingFrame); + } + + private readonly paintPendingFrame = () => { + this.animationFrame = 0; + const pending = this.pendingFrame; + this.pendingFrame = null; + if (!pending) { + return; + } + this.paintDecodedFrame(pending); + }; + + private paintDecodedFrame(pending: PendingVideoFrame) { + const canvas = this.canvas; + if (!canvas) { + pending.frame.close(); + return; + } + const videoFrame = pending.frame; + const width = + videoFrame.displayWidth ?? videoFrame.codedWidth ?? this.stats.width; + const height = + videoFrame.displayHeight ?? videoFrame.codedHeight ?? this.stats.height; + this.syncCanvasSize(width, height); + const startedAt = performance.now(); + try { + this.ensureCanvasContext()?.drawImage( + videoFrame as unknown as CanvasImageSource, + 0, + 0, + canvas.width, + canvas.height, + ); + } finally { + videoFrame.close(); + } + const finishedAt = performance.now(); + const previousFrameAt = this.lastFrameAt; + this.lastFrameAt = finishedAt; + this.stalledFrameWatchdogCount = 0; + this.reportVideoConfig(canvas.width, canvas.height); + this.stats.codec = "h264-ws"; + if (pending.sequence !== null) { + this.stats.frameSequence = pending.sequence; + } + this.stats.renderedFrames += 1; + this.stats.latestRenderMs = finishedAt - startedAt; + this.stats.maxRenderMs = Math.max( + this.stats.maxRenderMs, + this.stats.latestRenderMs, + ); + this.stats.averageRenderMs = + this.stats.averageRenderMs <= 0 + ? this.stats.latestRenderMs + : this.stats.averageRenderMs * 0.9 + this.stats.latestRenderMs * 0.1; + this.stats.latestFrameGapMs = + previousFrameAt > 0 ? finishedAt - previousFrameAt : 0; + this.stats.decodeQueueSize = this.decoder?.decodeQueueSize ?? 0; + this.stats.waitingForKeyFrame = false; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + if (!this.streamingReported) { + this.streamingReported = true; + this.onMessage({ + type: "status", + status: { + detail: "H264 WebSocket stream connected", + state: "streaming", + }, + }); + } + } + + private ensureCanvasContext(): CanvasRenderingContext2D | null { + const canvas = this.canvas; + if (!canvas) { + this.canvasContext = null; + return null; + } + if (this.canvasContext?.canvas === canvas) { + return this.canvasContext; + } + this.canvasContext = canvas.getContext("2d", { + alpha: false, + desynchronized: true, + }); + return this.canvasContext; + } + + private syncCanvasSize(width: number, height: number) { + if (!this.canvas) { + return; + } + const nextWidth = Math.max(1, Math.round(width)); + const nextHeight = Math.max(1, Math.round(height)); + if (this.canvas.width !== nextWidth) { + this.canvas.width = nextWidth; + } + if (this.canvas.height !== nextHeight) { + this.canvas.height = nextHeight; + } + } + + private reportVideoConfig(width: number, height: number) { + if ( + this.reportedVideoWidth === width && + this.reportedVideoHeight === height + ) { + return; + } + this.reportedVideoWidth = width; + this.reportedVideoHeight = height; + this.onMessage({ type: "video-config", size: { height, width } }); + } + + private scheduleFrameWatchdog(generation: number) { + this.clearFrameWatchdog(); + this.frameWatchdogTimeout = window.setTimeout( + () => { + this.frameWatchdogTimeout = 0; + if (generation !== this.connectGeneration || !this.shouldReconnect) { + return; + } + const now = performance.now(); + if (this.lastFrameAt <= 0) { + this.handleError("H264 WebSocket stream did not render a frame."); + return; + } + if (now - this.lastFrameAt > H264_WS_STALLED_FRAME_TIMEOUT_MS) { + this.stalledFrameWatchdogCount += 1; + this.sendControl({ snapshot: true, type: "streamControl" }); + this.sendControl({ forceKeyframe: true, type: "streamControl" }); + if (this.stalledFrameWatchdogCount >= 2 && this.streamTarget) { + const target = this.streamTarget; + this.onMessage({ + type: "status", + status: { + detail: "Reconnecting stalled H264 WebSocket stream", + state: "connecting", + }, + }); + void this.connect({ + ...target, + streamConfig: this.streamConfig, + }); + return; + } + } else { + this.stalledFrameWatchdogCount = 0; + } + this.scheduleFrameWatchdog(generation); + }, + this.lastFrameAt > 0 + ? H264_WS_STALLED_FRAME_TIMEOUT_MS + : H264_WS_FIRST_FRAME_TIMEOUT_MS, + ); + } + + private clearFrameWatchdog() { + if (!this.frameWatchdogTimeout) { + return; + } + window.clearTimeout(this.frameWatchdogTimeout); + this.frameWatchdogTimeout = 0; + } + + private closeDecoder() { + try { + this.decoder?.close(); + } catch { + // Closing a failed decoder is best effort. + } + this.decoder = null; + this.decoderConfigKey = ""; + this.pendingFrameSequences.clear(); + } + + private closePendingFrame() { + if (!this.pendingFrame) { + return; + } + try { + this.pendingFrame.frame.close(); + } catch { + // VideoFrame cleanup is best effort during disconnect/replacement. + } + this.pendingFrame = null; + } + + private takeRenderedFrameSequence( + videoFrame: WebCodecsVideoFrame, + ): number | null { + if (typeof videoFrame.timestamp !== "number") { + return null; + } + const sequence = this.pendingFrameSequences.get(videoFrame.timestamp); + this.pendingFrameSequences.delete(videoFrame.timestamp); + return typeof sequence === "number" ? sequence : null; + } + + private trimPendingFrameSequenceMap() { + while (this.pendingFrameSequences.size > 256) { + const firstKey = this.pendingFrameSequences.keys().next().value; + if (typeof firstKey !== "number") { + break; + } + this.pendingFrameSequences.delete(firstKey); + } + } + + private recordH264SocketMessage(data: unknown) { + this.stats.h264SocketMessages += 1; + const byteLength = + typeof data === "string" + ? data.length + : typeof (data as { byteLength?: unknown })?.byteLength === "number" + ? ((data as { byteLength: number }).byteLength ?? 0) + : typeof (data as { size?: unknown })?.size === "number" + ? ((data as { size: number }).size ?? 0) + : 0; + this.stats.h264SocketBytes += Math.max(0, byteLength); + this.stats.h264SocketMessageType = Object.prototype.toString.call(data); + } + + private startAdaptiveQuality(generation: number) { + this.clearAdaptiveQuality(); + if (this.streamConfig?.quality !== "auto") { + return; + } + this.adaptiveLastAt = performance.now(); + this.adaptiveLastDecodedFrames = this.stats.decodedFrames; + this.adaptiveLastRenderedFrames = this.stats.renderedFrames; + this.adaptiveInterval = window.setInterval(() => { + if (generation !== this.connectGeneration || !this.shouldReconnect) { + this.clearAdaptiveQuality(); + return; + } + void this.sampleAdaptiveQuality(); + }, H264_WS_ADAPTIVE_SAMPLE_MS); + } + + private clearAdaptiveQuality() { + if (!this.adaptiveInterval) { + return; + } + window.clearInterval(this.adaptiveInterval); + this.adaptiveInterval = 0; + } + + private async sampleAdaptiveQuality() { + if (this.streamConfig?.quality !== "auto") { + this.clearAdaptiveQuality(); + return; + } + const now = performance.now(); + const elapsedSeconds = Math.max((now - this.adaptiveLastAt) / 1000, 0.001); + const renderedDelta = + this.stats.renderedFrames - this.adaptiveLastRenderedFrames; + const decodedDelta = + this.stats.decodedFrames - this.adaptiveLastDecodedFrames; + this.adaptiveLastAt = now; + this.adaptiveLastRenderedFrames = this.stats.renderedFrames; + this.adaptiveLastDecodedFrames = this.stats.decodedFrames; + + const renderedFps = renderedDelta / elapsedSeconds; + const decodedFps = decodedDelta / elapsedSeconds; + const underPressure = + this.stats.decodeQueueSize > 1 || + this.stats.latestFrameGapMs > 80 || + this.stats.averageRenderMs > 4; + if (underPressure) { + this.autoProfileStableSamples = 0; + await this.setAutoProfile(this.nextLowerAutoProfile()); + return; + } + + if ( + renderedFps > 0 && + decodedFps > 0 && + this.stats.decodeQueueSize === 0 && + this.stats.latestFrameGapMs < 40 && + this.stats.averageRenderMs < 2 + ) { + this.autoProfileStableSamples += 1; + } else { + this.autoProfileStableSamples = 0; + } + if ( + this.autoProfileStableSamples >= H264_WS_AUTO_STABLE_SAMPLES_TO_UPGRADE + ) { + this.autoProfileStableSamples = 0; + await this.setAutoProfile(this.nextHigherAutoProfile()); + } + } + + private nextLowerAutoProfile(): StreamQualityPreset { + const profiles = h264AutoProfiles(this.streamTarget); + const index = profiles.indexOf(this.autoProfile); + return profiles[Math.max(0, index - 1)] ?? this.autoProfile; + } + + private nextHigherAutoProfile(): StreamQualityPreset { + const profiles = h264AutoProfiles(this.streamTarget); + const index = profiles.indexOf(this.autoProfile); + return ( + profiles[Math.min(profiles.length - 1, Math.max(0, index) + 1)] ?? + this.autoProfile + ); + } + + private async setAutoProfile(profile: StreamQualityPreset) { + if (profile === this.autoProfile || this.streamConfig?.quality !== "auto") { + return; + } + this.autoProfile = profile; + const effectiveConfig = h264WebSocketStreamConfig( + this.streamConfig, + this.autoProfile, + ); + if (!effectiveConfig) { + return; + } + if (!sendStreamQualityConfig(effectiveConfig)) { + await postStreamConfigWithAuthRetry(effectiveConfig, { + remote: this.streamTarget?.remote, + }).catch(() => { + // Stream-quality adaptation is opportunistic; the stream socket handles reachability. + }); + } + this.sendControl({ forceKeyframe: true, type: "streamControl" }); + } + + private handleError(message: string) { + this.onMessage({ + type: "status", + status: { error: message.replace(/\.$/, ""), state: "error" }, + }); + } +} + +function parseH264WebSocketFrame(data: unknown): H264WebSocketFrame | null { + const bytes = bytesFromBinaryMessage(data); + if (!bytes || bytes.byteLength < H264_WS_HEADER_BYTES) { + return null; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (view.getUint32(0, false) !== H264_WS_MAGIC || view.getUint8(4) !== 1) { + return null; + } + const flags = view.getUint8(5); + const headerBytes = view.getUint16(6, false); + if (headerBytes < H264_WS_HEADER_BYTES || headerBytes > bytes.byteLength) { + return null; + } + const configBytes = view.getUint32(32, false); + const payloadBytes = view.getUint32(36, false); + const payloadOffset = headerBytes + configBytes; + const payloadEnd = payloadOffset + payloadBytes; + if (payloadEnd > bytes.byteLength) { + return null; + } + const config = + flags & H264_WS_FLAG_CONFIG + ? bytes.subarray(headerBytes, payloadOffset) + : new Uint8Array(); + return { + config, + height: view.getUint32(28, false), + isKeyFrame: Boolean(flags & H264_WS_FLAG_KEYFRAME), + payload: bytes.subarray(payloadOffset, payloadEnd), + sequence: Number(view.getBigUint64(8, false)), + timestampUs: Number(view.getBigUint64(16, false)), + width: view.getUint32(24, false), + }; +} + +function bytesFromBinaryMessage(data: unknown): Uint8Array | null { + if (ArrayBuffer.isView(data)) { + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + } + if (data instanceof ArrayBuffer) { + return new Uint8Array(data); + } + if ( + Object.prototype.toString.call(data) === "[object ArrayBuffer]" && + typeof (data as { byteLength?: unknown }).byteLength === "number" + ) { + return new Uint8Array(data as ArrayBuffer); + } + if ( + typeof data === "object" && + data !== null && + typeof (data as { byteLength?: unknown }).byteLength === "number" + ) { + try { + return new Uint8Array(data as ArrayBufferLike); + } catch { + return null; + } + } + return null; +} + +function h264CodecStringFromAvcC(config: Uint8Array): string | null { + if (config.byteLength < 4 || config[0] !== 1) { + return null; + } + return `avc1.${hexByte(config[1])}${hexByte(config[2])}${hexByte(config[3])}`; +} + +function h264DecoderConfigKey(frame: H264WebSocketFrame): string { + const codec = h264CodecStringFromAvcC(frame.config) ?? "avc1.42E01F"; + const prefix = frame.config.byteLength + ? Array.from(frame.config.slice(0, 16), hexByte).join("") + : ""; + return `${codec}:${frame.width}x${frame.height}:${frame.config.byteLength}:${prefix}`; +} + +function arrayBufferCopy(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function hexByte(byte: number): string { + return byte.toString(16).padStart(2, "0").toUpperCase(); +} + class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; @@ -218,6 +1228,15 @@ class WebRtcStreamClient implements StreamClientBackend { attachCanvas(canvasElement: HTMLCanvasElement) { this.canvas = canvasElement; + if ( + this.video && + this.video.parentElement !== canvasElement.parentElement + ) { + canvasElement.parentElement?.insertBefore( + this.video, + canvasElement.nextSibling, + ); + } } clear() { @@ -289,18 +1308,6 @@ class WebRtcStreamClient implements StreamClientBackend { }); try { - try { - await postStreamConfigWithAuthRetry(target.streamConfig, { - remote: target.remote, - }); - } catch (error) { - if (!target.remote) { - throw error; - } - } - if (generation !== this.connectGeneration) { - return; - } const health = await fetchHealth().catch(() => null); if (generation !== this.connectGeneration) { return; @@ -371,10 +1378,8 @@ class WebRtcStreamClient implements StreamClientBackend { (video as HTMLVideoElement & { latencyHint?: string }).latencyHint = "interactive"; video.srcObject = stream; - canvasElement.parentElement?.insertBefore( - video, - canvasElement.nextSibling, - ); + const mountCanvas = this.canvas ?? canvasElement; + mountCanvas.parentElement?.insertBefore(video, mountCanvas.nextSibling); this.video = video; const startPlayback = () => { if (generation !== this.connectGeneration) { @@ -478,13 +1483,10 @@ class WebRtcStreamClient implements StreamClientBackend { return; } const generation = ++this.streamConfigGeneration; - await postStreamConfigWithAuthRetry(config, { remote: this.remoteMode }); - if (generation !== this.streamConfigGeneration) { - return; + if (!sendStreamQualityConfig(config)) { + await postStreamConfigWithAuthRetry(config, { remote: this.remoteMode }); } - const target = this.streamTarget; - if (target && this.shouldReconnect) { - await this.connect({ ...target, streamConfig: config }); + if (generation !== this.streamConfigGeneration) { return; } this.sendControl({ forceKeyframe: true, type: "streamControl" }); @@ -503,7 +1505,10 @@ class WebRtcStreamClient implements StreamClientBackend { return; } await peerConnection.setLocalDescription(offer); - await waitForIceGathering(peerConnection); + await waitForIceGathering(peerConnection, { + resolveOnHostCandidate: !target.remote, + timeoutMs: target.remote ? 3000 : 250, + }); if (generation !== this.connectGeneration) { return; } @@ -515,6 +1520,11 @@ class WebRtcStreamClient implements StreamClientBackend { localDescription.sdp, ); this.postDiagnostics(target, `${options.detailPrefix}-offer`); + if (target.remote && !sdpHasCandidateType(localDescription.sdp, "host")) { + throw new Error( + "WebRTC gathered no host ICE candidates for this remote browser.", + ); + } const response = await postWebRtcOfferWithAuthRetry( target, @@ -951,20 +1961,9 @@ class WebRtcStreamClient implements StreamClientBackend { url: window.location.href, userAgent: window.navigator.userAgent, }; - if (sendWebRtcClientStats(payload) || this.remoteMode) { + if (sendStreamClientStats(payload) || this.remoteMode) { return; } - void fetch( - new URL(apiUrl("/api/client-stream-stats"), window.location.href), - { - body: JSON.stringify(payload), - cache: "no-store", - headers: apiHeaders(), - method: "POST", - }, - ).catch(() => { - // Diagnostics only. - }); } private drawVideoFrame = () => { @@ -1241,16 +2240,36 @@ async function postStreamConfigWithAuthRetry( function postStreamConfig(config: StreamConfig): Promise { return fetch(apiUrl("/api/stream-quality"), { - body: JSON.stringify({ - fps: config.fps, - profile: config.quality, - videoCodec: config.encoder, - }), + body: JSON.stringify(streamQualityPayload(config)), headers: apiHeaders(), method: "POST", }); } +function streamQualityPayload(config: StreamConfig): { + fps: number; + profile: StreamQualityPreset; + videoCodec: string; +} { + return { + fps: config.fps, + profile: config.quality === "auto" ? "economy" : config.quality, + videoCodec: config.encoder, + }; +} + +function streamQualityQuery(config: StreamConfig | undefined): string { + if (!config) { + return ""; + } + const params = new URLSearchParams(); + const payload = streamQualityPayload(config); + params.set("fps", String(payload.fps)); + params.set("profile", payload.profile); + params.set("videoCodec", payload.videoCodec); + return `?${params.toString()}`; +} + function postWebRtcOffer( target: StreamConnectTarget, localDescription: RTCSessionDescription, @@ -1261,6 +2280,9 @@ function postWebRtcOffer( body: JSON.stringify({ clientId: target.clientId, sdp: localDescription.sdp, + streamConfig: target.streamConfig + ? streamQualityPayload(target.streamConfig) + : undefined, type: localDescription.type, }), headers: apiHeaders(), @@ -1269,6 +2291,49 @@ function postWebRtcOffer( ); } +function h264WebSocketStreamConfig( + config: StreamConfig | undefined, + autoProfile: StreamQualityPreset, +): StreamConfig | undefined { + if (!config) { + return config; + } + if (config.quality === "auto") { + return { + ...config, + fps: Math.min(config.fps || 60, 60), + maxEdge: undefined, + quality: autoProfile, + }; + } + if (config.quality !== "quality") { + return config; + } + return config; +} + +function h264AutoProfiles( + target: StreamConnectTarget | null, +): StreamQualityPreset[] { + return shouldUseRemoteH264AutoProfile(target) + ? H264_WS_REMOTE_AUTO_PROFILES + : H264_WS_LOCAL_AUTO_PROFILES; +} + +function initialH264AutoProfile( + target: StreamConnectTarget | null, +): StreamQualityPreset { + return shouldUseRemoteH264AutoProfile(target) + ? H264_WS_REMOTE_INITIAL_AUTO_PROFILE + : H264_WS_LOCAL_INITIAL_AUTO_PROFILE; +} + +function shouldUseRemoteH264AutoProfile( + target: StreamConnectTarget | null, +): boolean { + return Boolean(target?.remote) || !isLoopbackHost(window.location.hostname); +} + function configureLowLatencyReceiver( receiver: RTCRtpReceiver, bufferSeconds: number | null, @@ -1413,6 +2478,21 @@ function summarizeSdpCandidates(sdp: string): string { ); } +function sdpHasCandidateType(sdp: string, candidateType: string): boolean { + return sdp + .split(/\r?\n/) + .filter((line) => line.startsWith("a=candidate:")) + .some((line) => + candidateLineHasType(line.slice("a=".length), candidateType), + ); +} + +function candidateLineHasType(line: string, candidateType: string): boolean { + const parts = line.split(/\s+/); + const typIndex = parts.indexOf("typ"); + return typIndex >= 0 && parts[typIndex + 1] === candidateType; +} + function summarizeCandidateLines(lines: string[]): string { const counts: Record = { host: 0, @@ -1454,18 +2534,51 @@ function candidateStatsSummary(candidate: RTCStats | undefined): string { return `${stats.candidateType ?? "?"}/${stats.protocol ?? "?"}/${stats.address || stats.ip ? "addr" : "noaddr"}/${stats.port ?? "?"}`; } -function waitForIceGathering(peerConnection: RTCPeerConnection) { - if (peerConnection.iceGatheringState === "complete") { +function waitForIceGathering( + peerConnection: RTCPeerConnection, + options: { resolveOnHostCandidate?: boolean; timeoutMs?: number } = {}, +) { + if ( + peerConnection.iceGatheringState === "complete" || + (options.resolveOnHostCandidate && + sdpHasCandidateType(peerConnection.localDescription?.sdp ?? "", "host")) + ) { return Promise.resolve(); } return new Promise((resolve) => { - const timeout = window.setTimeout(resolve, 3000); - peerConnection.addEventListener("icegatheringstatechange", () => { + let resolved = false; + const finish = () => { + if (resolved) { + return; + } + resolved = true; + window.clearTimeout(timeout); + peerConnection.removeEventListener( + "icegatheringstatechange", + handleGatheringStateChange, + ); + peerConnection.removeEventListener("icecandidate", handleIceCandidate); + resolve(); + }; + const handleGatheringStateChange = () => { if (peerConnection.iceGatheringState === "complete") { - window.clearTimeout(timeout); - resolve(); + finish(); } - }); + }; + const handleIceCandidate = (event: RTCPeerConnectionIceEvent) => { + if ( + options.resolveOnHostCandidate && + candidateLineHasType(event.candidate?.candidate ?? "", "host") + ) { + finish(); + } + }; + const timeout = window.setTimeout(finish, options.timeoutMs ?? 3000); + peerConnection.addEventListener( + "icegatheringstatechange", + handleGatheringStateChange, + ); + peerConnection.addEventListener("icecandidate", handleIceCandidate); }); } @@ -1473,7 +2586,10 @@ export class StreamWorkerClient { private readonly onMessage: (message: WorkerToMainMessage) => void; private backend: StreamClientBackend | null = null; private attachedCanvas = false; + private backendKind: StreamBackend | null = null; + private canvasElement: HTMLCanvasElement | null = null; private disposed = false; + private target: StreamConnectTarget | null = null; private readonly destroyOnPageExit = () => { this.destroy(); }; @@ -1487,17 +2603,16 @@ export class StreamWorkerClient { } attachCanvas(canvasElement: HTMLCanvasElement) { - if (this.attachedCanvas) { - return; - } - - this.backend = new WebRtcStreamClient(this.onMessage); - this.backend.attachCanvas(canvasElement); + this.canvasElement = canvasElement; this.attachedCanvas = true; + this.backend?.attachCanvas(canvasElement); } connect(target: StreamConnectTarget) { try { + this.target = target; + const backendKind = initialStreamBackend(target); + this.setBackend(backendKind); const result = this.backend?.connect(target); if (result && typeof result.catch === "function") { result.catch((error: unknown) => { @@ -1523,6 +2638,7 @@ export class StreamWorkerClient { disconnect() { this.backend?.disconnect(); + this.target = null; } clear() { @@ -1565,4 +2681,78 @@ export class StreamWorkerClient { activeStreamClient = null; } } + + private setBackend(kind: StreamBackend) { + if (this.backend && this.backendKind === kind) { + return; + } + this.backend?.destroy(); + this.backend = + kind === "h264-ws" + ? new H264WebSocketStreamClient(this.handleBackendMessage) + : new WebRtcStreamClient(this.handleBackendMessage); + this.backendKind = kind; + if (this.canvasElement) { + this.backend.attachCanvas(this.canvasElement); + } + } + + private readonly handleBackendMessage = (message: WorkerToMainMessage) => { + if ( + message.type === "status" && + message.status.state === "error" && + preferredStreamBackend(this.target) === "auto" && + this.target + ) { + const nextBackend = nextAutoFallbackBackend(this.backendKind); + if (!nextBackend) { + this.onMessage(message); + return; + } + const target = this.target; + this.setBackend(nextBackend); + this.onMessage({ + type: "status", + status: { + detail: "Falling back to H264 WebSocket", + state: "connecting", + }, + }); + this.backend?.connect(target); + return; + } + this.onMessage(message); + }; +} + +function preferredStreamBackend( + target?: StreamConnectTarget | null, +): "auto" | StreamBackend { + const value = + target?.transport ?? + new URLSearchParams(window.location.search).get("stream"); + if (value === "h264" || value === "h264-ws") { + return "h264-ws"; + } + return value === "webrtc" ? "webrtc" : "auto"; +} + +function initialStreamBackend(target: StreamConnectTarget): StreamBackend { + const preferredBackend = preferredStreamBackend(target); + if (preferredBackend === "h264-ws") { + return canUseH264WebSocket() ? "h264-ws" : "webrtc"; + } + if (preferredBackend === "webrtc") { + return canUseWebRtc() ? "webrtc" : "h264-ws"; + } + return canUseWebRtc() ? "webrtc" : "h264-ws"; +} + +function nextAutoFallbackBackend( + current: StreamBackend | null, +): StreamBackend | null { + if (current === "webrtc") { + return canUseH264WebSocket() ? "h264-ws" : null; + } + return null; } diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index d3eccf5a..55f40118 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -7,8 +7,7 @@ import type { Size } from "../viewport/types"; import { createEmptyStreamStats } from "./stats"; import { buildStreamTarget, - canUseWebRtc, - sendWebRtcClientStats, + sendStreamClientStats, StreamWorkerClient, type StreamBackend, type VisualArtifactSample, @@ -18,6 +17,7 @@ import type { StreamConfig, StreamStats, StreamStatus, + StreamTransport, WorkerToMainMessage, } from "./streamTypes"; @@ -25,7 +25,7 @@ const FPS_SAMPLE_INTERVAL_MS = 1000; const CLIENT_TELEMETRY_INTERVAL_MS = 1000; const REMOTE_CLIENT_TELEMETRY_INTERVAL_MS = 5000; const CLIENT_TELEMETRY_ID_STORAGE_KEY = "simdeck.streamClientId"; -const VISUAL_ARTIFACT_TELEMETRY_INTERVAL_MS = 1000; +const VISUAL_ARTIFACT_TELEMETRY_INTERVAL_MS = 30000; interface UseLiveStreamOptions { canvasElement: HTMLCanvasElement | null; @@ -34,6 +34,7 @@ interface UseLiveStreamOptions { simulator: SimulatorMetadata | null; streamConfig?: StreamConfig; streamConfigApplyKey?: number; + streamTransport?: StreamTransport; } interface UseLiveStreamResult { @@ -88,6 +89,16 @@ function buildClientTelemetryUrl(): string { ).toString(); } +function currentClientBundle(): string { + return ( + Array.from(document.scripts) + .map((script) => script.src) + .find((src) => /\/assets\/index-[^/]+\.js(?:$|\?)/.test(src)) + ?.split("/") + .pop() ?? "" + ); +} + export function useLiveStream({ canvasElement, paused = false, @@ -95,6 +106,7 @@ export function useLiveStream({ simulator, streamConfig, streamConfigApplyKey = 0, + streamTransport = "auto", }: UseLiveStreamOptions): UseLiveStreamResult { const clientTelemetryIdRef = useRef(""); const workerClientRef = useRef(null); @@ -103,6 +115,9 @@ export function useLiveStream({ const latestRenderedFramesRef = useRef(0); const latestStatsRef = useRef(createEmptyStreamStats()); const latestStatusRef = useRef({ state: "idle" }); + const retainedFrameRef = useRef(false); + const previousSimulatorUdidRef = useRef(simulator?.udid); + const connectedStreamTargetKeyRef = useRef(""); const latestVisualArtifactRef = useRef(null); const latestVisualArtifactSampleCountRef = useRef(0); const lastVisualArtifactSampleAtRef = useRef(0); @@ -144,13 +159,20 @@ export function useLiveStream({ }, []); useEffect(() => { - if (paused || !canvasElement || workerClientRef.current) { + if (!canvasElement) { return; } - const workerClient = new StreamWorkerClient( - (message: WorkerToMainMessage) => { + let workerClient = workerClientRef.current; + if (!workerClient) { + workerClient = new StreamWorkerClient((message: WorkerToMainMessage) => { if (message.type === "stats") { + if ( + message.stats.decodedFrames > 0 || + message.stats.renderedFrames > 0 + ) { + retainedFrameRef.current = true; + } setStats(message.stats); return; } @@ -170,12 +192,12 @@ export function useLiveStream({ } setDeviceNaturalSize(message.size); - }, - ); + }); + workerClientRef.current = workerClient; + } try { workerClient.attachCanvas(canvasElement); - workerClientRef.current = workerClient; } catch (attachError) { const message = attachError instanceof Error @@ -184,25 +206,18 @@ export function useLiveStream({ setError(message); setStatus({ error: message, state: "error" }); workerClient.destroy(); + workerClientRef.current = null; return; } + }, [canvasElement]); - const destroyOnPageHide = () => { - workerClient.destroy(); - if (workerClientRef.current === workerClient) { - workerClientRef.current = null; - } - }; - window.addEventListener("pagehide", destroyOnPageHide); - window.addEventListener("beforeunload", destroyOnPageHide); - + useEffect(() => { return () => { - window.removeEventListener("pagehide", destroyOnPageHide); - window.removeEventListener("beforeunload", destroyOnPageHide); - workerClient.destroy(); + workerClientRef.current?.destroy(); workerClientRef.current = null; + connectedStreamTargetKeyRef.current = ""; }; - }, [canvasElement, paused]); + }, []); useEffect(() => { latestDecodedFramesRef.current = stats.decodedFrames; @@ -219,7 +234,16 @@ export function useLiveStream({ }, [fps]); useEffect(() => { - setStreamCanvasRevision((current) => current + 1); + const previousUdid = previousSimulatorUdidRef.current; + const nextUdid = simulator?.udid; + if (previousUdid === nextUdid) { + return; + } + previousSimulatorUdidRef.current = nextUdid; + retainedFrameRef.current = false; + if (previousUdid && nextUdid) { + setStreamCanvasRevision((current) => current + 1); + } }, [simulator?.udid]); useEffect(() => { @@ -256,41 +280,55 @@ export function useLiveStream({ useEffect(() => { const workerClient = workerClientRef.current; - if (!workerClient) { + if (!workerClient || !canvasElement) { return; } - setDeviceNaturalSize(null); - setStats(createEmptyStreamStats()); - setStatus({ state: "idle" }); - setError(""); - setFps(0); - if (paused || !simulator?.isBooted) { - workerClient.disconnect(); + setDeviceNaturalSize(null); + setStats(createEmptyStreamStats()); + setStatus({ state: "idle" }); + setError(""); + setFps(0); + retainedFrameRef.current = false; + if (connectedStreamTargetKeyRef.current) { + workerClient.disconnect(); + } + connectedStreamTargetKeyRef.current = ""; workerClient.clear(); return; } - if (!canUseWebRtc()) { - setStatus({ - error: "This browser does not support WebRTC video.", - state: "error", - }); + const targetKey = [ + simulator.udid, + remote ? "remote" : "local", + streamTransport, + ].join("|"); + if (connectedStreamTargetKeyRef.current === targetKey) { return; } - + setDeviceNaturalSize(null); + setStats(createEmptyStreamStats()); + setStatus({ state: "idle" }); + setError(""); + setFps(0); + connectedStreamTargetKeyRef.current = targetKey; workerClient.connect( buildStreamTarget(simulator.udid, { clientId: clientTelemetryIdRef.current, remote, streamConfig, + transport: streamTransport, }), ); - return () => { - workerClient.disconnect(); - }; - }, [canvasElement, simulator?.isBooted, simulator?.udid, paused, remote]); + }, [ + canvasElement, + simulator?.isBooted, + simulator?.udid, + paused, + remote, + streamTransport, + ]); useEffect(() => { if ( @@ -348,6 +386,7 @@ export function useLiveStream({ udid: simulator.udid, url: window.location.href, userAgent: window.navigator.userAgent, + clientBundle: currentClientBundle(), visualBadPixelRatio: latestVisualArtifact?.badPixelRatio, visualMaxPixelDiff: latestVisualArtifact?.maxPixelDiff, visualMaxTileDiff: latestVisualArtifact?.maxTileMeanDiff, @@ -355,7 +394,13 @@ export function useLiveStream({ visualSampleCount: latestVisualArtifactSampleCountRef.current, visibilityState: document.visibilityState, }; - if (sendWebRtcClientStats(payload) || remote) { + if ( + sendStreamClientStats(payload) || + remote || + streamTransport === "h264" || + streamTransport === "webrtc" || + streamTransport === "auto" + ) { return; } void fetch(buildClientTelemetryUrl(), { @@ -376,17 +421,20 @@ export function useLiveStream({ return () => { window.clearInterval(intervalId); }; - }, [remote, simulator?.udid]); + }, [remote, simulator?.udid, streamTransport]); return { deviceNaturalSize, error, fps, - hasFrame: status.state === "streaming" || stats.decodedFrames > 0, + hasFrame: + status.state === "streaming" || + stats.decodedFrames > 0 || + retainedFrameRef.current, runtimeInfo, stats, status, - streamBackend: "webrtc", - streamCanvasKey: `webrtc-${streamCanvasRevision}`, + streamBackend: stats.codec === "h264-ws" ? "h264-ws" : "webrtc", + streamCanvasKey: `stream-${streamCanvasRevision}`, }; } diff --git a/client/src/features/toolbar/Toolbar.tsx b/client/src/features/toolbar/Toolbar.tsx index 00baccd7..5606c235 100644 --- a/client/src/features/toolbar/Toolbar.tsx +++ b/client/src/features/toolbar/Toolbar.tsx @@ -6,6 +6,7 @@ import type { StreamEncoder, StreamFps, StreamQualityPreset, + StreamTransport, } from "../stream/streamTypes"; import { SimulatorMenu } from "../simulators/SimulatorMenu"; @@ -29,6 +30,7 @@ interface ToolbarProps { onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; onStreamQualityChange: (quality: StreamQualityPreset) => void; + onStreamTransportChange: (transport: StreamTransport) => void; onToggleAppearance: () => void; onToggleDebug: () => void; onToggleHierarchy: () => void; @@ -42,6 +44,7 @@ interface ToolbarProps { showBootButton: boolean; showStopButton: boolean; streamConfig: StreamConfig; + streamTransport: StreamTransport; touchOverlayVisible: boolean; menuOpen: boolean; menuRef: RefObject; @@ -71,6 +74,7 @@ export function Toolbar({ onStreamEncoderChange, onStreamFpsChange, onStreamQualityChange, + onStreamTransportChange, onToggleAppearance, onToggleDebug, onToggleHierarchy, @@ -84,6 +88,7 @@ export function Toolbar({ showBootButton, showStopButton, streamConfig, + streamTransport, touchOverlayVisible, }: ToolbarProps) { const [errorCopied, setErrorCopied] = useState(false); @@ -133,6 +138,7 @@ export function Toolbar({ onStreamEncoderChange={onStreamEncoderChange} onStreamFpsChange={onStreamFpsChange} onStreamQualityChange={onStreamQualityChange} + onStreamTransportChange={onStreamTransportChange} onToggleAppearance={onToggleAppearance} onToggleDebug={onToggleDebug} onToggleMenu={onToggleMenu} @@ -142,6 +148,7 @@ export function Toolbar({ selectedSimulator={selectedSimulator} setSelectedUDID={setSelectedUDID} streamConfig={streamConfig} + streamTransport={streamTransport} touchOverlayVisible={touchOverlayVisible} /> {selectedSimulator ? ( diff --git a/client/src/styles/components.css b/client/src/styles/components.css index c1f26dca..184b3a80 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -357,6 +357,40 @@ grid-template-columns: repeat(2, minmax(0, 1fr)); } +.menu-field { + display: grid; + gap: 5px; +} + +.menu-field-label { + color: var(--text-muted); + font-size: 10px; + font-weight: 600; + letter-spacing: 0; + text-transform: uppercase; +} + +.menu-select { + width: 100%; + height: 32px; + padding: 0 28px 0 10px; + border: 1px solid var(--border-subtle); + border-radius: 6px; + background: var(--surface); + color: var(--text); + font: inherit; + font-size: 12px; +} + +.menu-select:hover { + background: var(--surface-hover); +} + +.menu-select:focus-visible { + outline: 2px solid color-mix(in srgb, var(--accent) 45%, transparent); + outline-offset: 2px; +} + .menu-option { min-width: 0; height: 30px; diff --git a/docs/api/health.md b/docs/api/health.md index 924c7571..aa8eaf84 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -117,7 +117,7 @@ from hardware to software encoding. ### Client stream stats -`client_streams` is a rolling buffer of the most recent reports a client posted to `POST /api/client-stream-stats`. The server keeps the last 48 entries per `(clientId, kind)` pair. +`client_streams` is a rolling buffer of the most recent reports from browser stream transports. WebRTC clients normally send reports over the telemetry data channel, H.264 WebSocket clients send them on the stream socket, and simple clients can still post to `POST /api/client-stream-stats`. The server keeps the last 48 entries per `(clientId, kind)` pair. The browser client uses these to render its in-app diagnostics overlay and to size its decoder workers. Every field is optional except `clientId` and `kind`; see [`ClientStreamStats`](https://github.com/NativeScript/SimDeck/blob/main/server/src/metrics/counters.rs) for the full schema. diff --git a/docs/api/rest.md b/docs/api/rest.md index f17dd73f..a0070e24 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -76,23 +76,25 @@ Returns the active stream encoder settings and available quality profiles. ### `POST /api/stream-quality` -Updates the active stream encoder settings for newly encoded frames. The browser -UI uses this before WebRTC negotiation when the user selects encoder, FPS, or -quality. +Updates the active stream encoder settings for newly encoded frames. Browser +clients normally send these updates on the active WebRTC data channel or H.264 +WebSocket; this endpoint remains for scripts and fallback clients. ```json { "videoCodec": "hardware", - "fps": 120, - "profile": "quality" + "fps": 60, + "profile": "full" } ``` `videoCodec` accepts `hardware` or `software` from the UI, and the API also accepts `auto`. `fps` is clamped to the local stream range. Browser viewers show -five profiles: `quality` (4096 px), `balanced` (1280 px), `economy` (1080 px), -`low` (720 px), and `tiny` (540 px). The API still accepts the legacy `fast`, -`smooth`, and `ci-software` profiles for CLI/provider compatibility. When +five H.264 resolution profiles: `full` (4096 px at 60 fps), `balanced` +(1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), +and `tiny` (540 px at 30 fps). The API still accepts the legacy `quality`, +`fast`, `smooth`, and `ci-software` profiles for CLI/provider compatibility. +When `profile` is provided, its resolution preset is applied; send `maxEdge` without `profile` for a custom resolution cap. @@ -164,6 +166,11 @@ and the server responds with an SDP answer for a receive-only H.264 video track: ```json { "sdp": "v=0\r\n...", + "streamConfig": { + "fps": 60, + "profile": "full", + "videoCodec": "auto" + }, "type": "offer" } ``` @@ -178,15 +185,64 @@ and the server responds with an SDP answer for a receive-only H.264 video track: The endpoint requires the active simulator stream to produce H.264-compatible samples. The bundled browser client always uses this endpoint. -The browser also opens a `simdeck-control` data channel. In addition to input -messages, clients can tune the stream attached to that peer: +The browser also opens `simdeck-control` and `simdeck-telemetry` data channels. +In addition to input messages, clients can request a keyframe or tune the +stream attached to that peer: ```json -{ "type": "streamControl", "profile": "thumb" } +{ "type": "streamControl", "forceKeyframe": true } ``` -Supported profiles are `thumb`/`thumbnail`, `focus`/`full`, and `paused`. -Clients may also send `fps`, `forceKeyframe`, or `snapshot` fields. +```json +{ "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } +``` + +The telemetry channel accepts: + +```json +{ "type": "clientStats", "stats": { "clientId": "browser", "kind": "webrtc" } } +``` + +### `GET /api/simulators/{udid}/h264` + +Direct H.264 video over WebSocket for browsers that support WebCodecs but +cannot establish WebRTC media. The server sends binary messages with this +layout: + +| Offset | Size | Field | +| ------ | ---- | --------------------------------------------------- | +| 0 | 4 | Magic bytes `SDH1` | +| 4 | 1 | Version, currently `1` | +| 5 | 1 | Flags: bit 0 keyframe, bit 1 decoder config present | +| 6 | 2 | Header length, big-endian | +| 8 | 8 | Frame sequence, big-endian | +| 16 | 8 | Timestamp in microseconds, big-endian | +| 24 | 4 | Encoded width, big-endian | +| 28 | 4 | Encoded height, big-endian | +| 32 | 4 | Decoder config byte length, big-endian | +| 36 | 4 | H.264 sample byte length, big-endian | + +The optional decoder config bytes follow the header, then the encoded H.264 +sample bytes. Clients can pass initial stream settings as query parameters +(`profile`, `fps`, `videoCodec`) and can send text control messages on the same +socket: + +```json +{ "type": "streamControl", "forceKeyframe": true } +``` + +```json +{ "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } +``` + +```json +{ "type": "clientStats", "stats": { "clientId": "browser", "kind": "page" } } +``` + +Touch and keyboard input should use the separate `/api/simulators/{udid}/input` +WebSocket. The video socket is latest-frame oriented: clients should paint the +latest decoded frame locally and request a keyframe if the decoder loses sync, +rather than ACKing every rendered frame. ### `POST /api/simulators/{udid}/open-url` diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 466c3701..ecfad846 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -28,17 +28,17 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas `ui`, `daemon start`, and `daemon restart` accept the same server options. `ui` also accepts `--open`. -| Flag | Default | Description | -| -------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | -| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | -| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | -| `--stream-quality` | `smooth` | Realtime stream quality profile: `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +| Flag | Default | Description | +| -------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | +| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | +| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | +| `--client-root` | bundled `client/dist` | Override the static browser client directory. | +| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | +| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | +| `--stream-quality` | `full` | Realtime stream quality profile: `full`, `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | +| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | +| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | `studio expose` defaults to software H.264. Pass `--video-codec hardware` to opt into the hardware encoder when that is preferable. diff --git a/docs/contributing.md b/docs/contributing.md index 07fa8190..4ac2b6d6 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -70,7 +70,7 @@ If you contribute, keep these invariants in mind. They are also enforced by the - Don't add a Node or Swift dependency to solve work that already fits in Foundation/AppKit. - When touching private API usage, keep the adaptation small and explicit and document any simulator/runtime assumptions in `AGENTS.md`. - Prefer stable CLI subcommands over hidden environment variables. -- The supported live video path is the WebRTC H.264 offer endpoint. Do not bring back legacy `/stream.h264` handling. +- The supported live video paths are the WebRTC H.264 offer endpoint plus the `/api/simulators/{udid}/h264` WebSocket fallback. Do not bring back legacy `/stream.h264` handling. - If a feature depends on a booted simulator, fail with a clear JSON error instead of silently returning an empty asset. ## Linting and formatting diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index f84794bc..e2af3763 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -20,17 +20,17 @@ Owns the public CLI shape (`simdeck`, `simdeck ui`, `daemon`, `boot`, `shutdown` Key modules: -| Module | Responsibility | -| ----------------------------------- | -------------------------------------------------------------------------------------------- | -| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | -| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | -| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | -| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | -| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | -| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | -| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | -| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted via `/api/client-stream-stats`. | -| `server/src/logs.rs` | `os_log` log streaming and filtering. | +| Module | Responsibility | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | +| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | +| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | +| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | +| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | +| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | +| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | +| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | +| `server/src/logs.rs` | `os_log` log streaming and filtering. | The Rust server runs the tokio runtime on a worker thread while the AppKit main loop spins on the main thread. The native bridge needs the main loop to deliver display callbacks and HID events. @@ -119,4 +119,4 @@ If you contribute, keep the following invariants in mind: - Browser-only presentation logic stays in `client/`. - NativeScript app runtime inspection logic stays in `packages/nativescript-inspector/`. - Add a server endpoint before adding client-only assumptions. -- The supported live video path is the WebRTC H.264 offer endpoint. Do not bring back legacy `/stream.h264` handling. +- The supported live video paths are the WebRTC H.264 offer endpoint plus the `/api/simulators/{udid}/h264` WebSocket fallback. Do not bring back legacy `/stream.h264` handling. diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index c7553e77..9bb94502 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -54,17 +54,17 @@ This starts or reuses the project daemon, serves the bundled browser client, and `daemon start`, `daemon restart`, and `ui` accept the same server options: -| Flag | Default | Notes | -| -------------------- | --------------------- | --------------------------------------------------------------------------------- | -| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | -| `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). | -| `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | -| `--stream-quality` | `smooth` | Realtime stream quality profile, including `low`, `tiny`, and `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +| Flag | Default | Notes | +| -------------------- | --------------------- | ------------------------------------------------------------------------------------ | +| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | +| `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | +| `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | +| `--client-root` | bundled `client/dist` | Override the static browser client directory. | +| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). | +| `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | +| `--stream-quality` | `full` | Realtime stream quality profile, including `full`, `low`, `tiny`, and `ci-software`. | +| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | +| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | Example: diff --git a/docs/guide/video.md b/docs/guide/video.md index cb5700a4..a20989f5 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -1,6 +1,6 @@ # Video Pipeline -SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video playout. This page walks through the encoder choices, the keyframe handshake, and the metrics you can use to tune them. +SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video playout, with H.264 over WebSocket as the fallback for Safari and networks where peer media negotiation fails. This page walks through the encoder choices, fallback transport, keyframe handshake, and metrics you can use to tune them. ## Codec selection @@ -32,7 +32,7 @@ It is CLI-only because it is meant for less capable machines where freshness matters more than maximum smoothness. The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`. -The browser UI exposes stream controls for encoder, FPS, and five quality choices: `quality` (4096 px), `balanced` (1280 px), `economy` (1080 px), `low` (720 px), and `tiny` (540 px). Local browser sessions default to hardware H.264, 120 fps, and `quality`/full resolution with FPS choices of 30, 60, and 120. Remote browser sessions default to software H.264, 30 fps, and `balanced` with FPS choices of 15, 30, and 60. +The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality. ## Remote WebRTC ICE @@ -53,6 +53,47 @@ peer connection, so the local and remote peers use the same ICE configuration. Use `SIMDECK_WEBRTC_ICE_TRANSPORT_POLICY=all` or leave it unset for local LAN and localhost sessions. +## H264 WebSocket fallback + +The browser UI defaults to `?stream=auto`: it tries WebRTC first and falls back +to H264 over WebSocket if a decoded frame still does not render. +For remote browser sessions, SimDeck also falls back immediately when the +browser's WebRTC offer contains no local `host` ICE candidates, which covers +Safari privacy/network settings that suppress direct candidates. The stream +settings menu includes a transport picker for Auto, WebRTC, and H264 WS. You +can also force a mode while testing: + +```text +http://127.0.0.1:4310?stream=webrtc +http://127.0.0.1:4310?stream=h264 +``` + +H264 WS uses the same native H.264 encoder as WebRTC, but sends each encoded +sample on a binary WebSocket at: + +```http +GET /api/simulators/{udid}/h264?profile=full&fps=60&videoCodec=auto +``` + +Each message starts with a compact SimDeck header, followed by optional AVC +decoder config and the encoded sample bytes. The browser decodes with +WebCodecs, keeps only the latest decoded frame, and paints on +`requestAnimationFrame` so stale frames do not build latency. Input stays on +the separate `/api/simulators/{udid}/input` WebSocket so large video frames do +not block touch and keyboard messages. Stream-quality updates and client stats +use the WebRTC data channel or the H264 WS video socket instead of repeatedly +posting REST requests. H264 WS defaults to the `full` profile on loopback and +`auto` quality for remote sessions. H264 `Auto` starts at `full` on loopback; +remote `Auto` starts lower but can climb through the internal `smooth` step +(1170 px), `balanced`, and `full` after sustained low decode/render pressure. + +```http +GET /api/simulators/{udid}/input +``` + +That WebSocket accepts the same normalized control JSON used by the WebRTC data +channel and coalesces high-frequency touch `moved` events. + ## Keyframe handshake When a browser connects through `/api/simulators/{udid}/webrtc/offer`: @@ -78,12 +119,11 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques A few practical guidelines: -- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with the `quality` profile: full resolution, 120 fps, and a high bitrate floor. Pass `--video-codec software` only when the shared hardware encoder is unavailable or performs worse on that host. +- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Pass `--video-codec software` only when the shared hardware encoder is unavailable or performs worse on that host. - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. -- **Studio providers default to software H.264 plus `--stream-quality smooth`.** This profile uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. -- **Use `low` or `tiny` when resolution is the bottleneck.** `low` caps the longest edge at 720 pixels and targets 30 fps; `tiny` caps the longest edge at 540 pixels and targets 24 fps. -- **The remote browser renders the live stream as a native `