From 95212d13bc8ab51b0a3c3245c7a119140375ff8d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 7 May 2026 12:39:58 -0400 Subject: [PATCH 1/4] Add MJPEG fallback transport with WebSocket input --- README.md | 7 +- cli/XCWPrivateSimulatorSession.h | 9 + cli/XCWPrivateSimulatorSession.m | 233 ++++++++++ cli/native/XCWNativeBridge.h | 15 + cli/native/XCWNativeBridge.m | 13 + cli/native/XCWNativeSession.h | 4 + cli/native/XCWNativeSession.m | 46 ++ client/src/app/AppShell.tsx | 88 +++- .../src/features/simulators/SimulatorMenu.tsx | 32 +- client/src/features/stream/streamTypes.ts | 2 + .../src/features/stream/streamWorkerClient.ts | 409 +++++++++++++++++- client/src/features/stream/useLiveStream.ts | 26 +- client/src/features/toolbar/Toolbar.tsx | 7 + docs/guide/video.md | 41 +- server/src/api/routes.rs | 191 +++++++- server/src/native/bridge.rs | 16 + server/src/native/ffi.rs | 20 + server/src/simulators/session.rs | 124 +++++- server/src/transport/packet.rs | 11 + skills/simdeck/SKILL.md | 3 + 20 files changed, 1264 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 4273860f..1269508a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ view inside the editor. ## Features -- Local simulator video stream over browser-native WebRTC H.264 +- Local simulator video stream over browser-native WebRTC H.264 with direct HTTP MJPEG fallback - Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - CoreSimulator chrome asset rendering for device bezels @@ -68,8 +68,9 @@ LAN clients should pair with the printed code before receiving the API cookie. SimDeck Studio providers run the daemon on loopback and use `scripts/studio-provider-bridge.mjs` for outbound control-plane communication with Studio. Studio hosts the browser UI and proxies SimDeck REST requests over -that bridge while WebRTC media still negotiates directly between the browser and -runner through ICE. +that bridge while WebRTC media negotiates directly between the browser and +runner through ICE. If WebRTC fails before rendering a frame, the browser can +fall back to the daemon's direct MJPEG stream and WebSocket input channel. Expose a local simulator through SimDeck Studio with one command: diff --git a/cli/XCWPrivateSimulatorSession.h b/cli/XCWPrivateSimulatorSession.h index 90dc69af..17880608 100644 --- a/cli/XCWPrivateSimulatorSession.h +++ b/cli/XCWPrivateSimulatorSession.h @@ -10,6 +10,11 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData, NSData * _Nullable decoderConfig, CGSize dimensions); +typedef void (^XCWPrivateSimulatorJPEGFrameHandler)(NSData *jpegData, + NSUInteger frameSequence, + uint64_t timestampUs, + CGSize dimensions); + @interface XCWPrivateSimulatorSession : NSObject - (instancetype)init NS_UNAVAILABLE; @@ -34,6 +39,10 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData, - (NSDictionary *)videoEncoderStats; - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler; - (void)removeEncodedFrameListener:(id)token; +- (id)addJPEGFrameListenerWithMaxEdge:(NSUInteger)maxEdge + quality:(double)quality + handler:(XCWPrivateSimulatorJPEGFrameHandler)handler; +- (void)removeJPEGFrameListener:(id)token; - (BOOL)sendTouchWithNormalizedX:(double)normalizedX normalizedY:(double)normalizedY diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m index 18e9eac4..ea42e3fb 100644 --- a/cli/XCWPrivateSimulatorSession.m +++ b/cli/XCWPrivateSimulatorSession.m @@ -1,7 +1,10 @@ #import "XCWPrivateSimulatorSession.h" #import +#import #import +#import +#import #import "DFPrivateSimulatorDisplayBridge.h" #import "XCWH264Encoder.h" @@ -15,15 +18,129 @@ @interface XCWPrivateSimulatorSession () 0 ? data : nil; +} + @implementation XCWPrivateSimulatorSession { DFPrivateSimulatorDisplayBridge *_displayBridge; dispatch_queue_t _stateQueue; + dispatch_queue_t _jpegQueue; dispatch_semaphore_t _readinessSemaphore; XCWH264Encoder *_videoEncoder; NSString *_displayStatusValue; CGSize _displaySizeValue; NSMutableDictionary *_encodedFrameListeners; + NSMutableDictionary *_jpegFrameListeners; NSUInteger _encodedFrameSequenceValue; + NSUInteger _jpegFrameSequenceValue; + CVPixelBufferRef _pendingJPEGPixelBuffer; + BOOL _jpegEncodeInFlight; NSData *_latestKeyFrameData; uint64_t _latestKeyFrameTimestampUs; NSString *_latestKeyFrameCodec; @@ -62,8 +179,10 @@ - (nullable instancetype)initWithUDID:(NSString *)udid dispatch_queue_attr_t queueAttributes = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0); _stateQueue = dispatch_queue_create("com.simdeck.private-session.state", queueAttributes); + _jpegQueue = dispatch_queue_create("com.simdeck.private-session.jpeg", queueAttributes); _readinessSemaphore = dispatch_semaphore_create(0); _encodedFrameListeners = [NSMutableDictionary dictionary]; + _jpegFrameListeners = [NSMutableDictionary dictionary]; _displayStatusValue = bridge.displayStatus ?: @"Initializing private simulator display"; _displayReadyValue = bridge.isDisplayReady; __weak typeof(self) weakSelf = self; @@ -112,6 +231,10 @@ - (nullable instancetype)initWithUDID:(NSString *)udid - (void)dealloc { [_videoEncoder invalidate]; + if (_pendingJPEGPixelBuffer != nil) { + CVPixelBufferRelease(_pendingJPEGPixelBuffer); + _pendingJPEGPixelBuffer = nil; + } } - (BOOL)waitUntilReadyWithTimeout:(NSTimeInterval)timeout { @@ -159,6 +282,7 @@ - (void)refreshCurrentFrame { [self signalReadinessIfNeededLocked]; }); [_videoEncoder encodePixelBuffer:pixelBuffer]; + [self enqueueJPEGPixelBufferIfNeeded:pixelBuffer]; CVPixelBufferRelease(pixelBuffer); } @@ -205,6 +329,35 @@ - (void)removeEncodedFrameListener:(id)token { }); } +- (id)addJPEGFrameListenerWithMaxEdge:(NSUInteger)maxEdge + quality:(double)quality + handler:(XCWPrivateSimulatorJPEGFrameHandler)handler { + if (handler == nil) { + return [NSUUID UUID]; + } + + NSUUID *token = [NSUUID UUID]; + XCWPrivateSimulatorJPEGFrameListener *listener = [XCWPrivateSimulatorJPEGFrameListener new]; + listener.handler = handler; + listener.maxEdge = maxEdge; + listener.quality = quality; + dispatch_sync(_stateQueue, ^{ + self->_jpegFrameListeners[token] = listener; + }); + [self refreshCurrentFrame]; + return token; +} + +- (void)removeJPEGFrameListener:(id)token { + if (![token isKindOfClass:[NSUUID class]]) { + return; + } + + dispatch_sync(_stateQueue, ^{ + [self->_jpegFrameListeners removeObjectForKey:(NSUUID *)token]; + }); +} + - (BOOL)isDisplayReady { __block BOOL ready = NO; dispatch_sync(_stateQueue, ^{ @@ -334,6 +487,7 @@ - (void)privateSimulatorDisplayBridge:(DFPrivateSimulatorDisplayBridge *)bridge [self signalReadinessIfNeededLocked]; }); [_videoEncoder encodePixelBuffer:pixelBuffer]; + [self enqueueJPEGPixelBufferIfNeeded:pixelBuffer]; } - (void)privateSimulatorDisplayBridge:(DFPrivateSimulatorDisplayBridge *)bridge @@ -363,10 +517,89 @@ - (void)primeStateFromBridge { [self signalReadinessIfNeededLocked]; }); [_videoEncoder encodePixelBuffer:pixelBuffer]; + [self enqueueJPEGPixelBufferIfNeeded:pixelBuffer]; CVPixelBufferRelease(pixelBuffer); } } +- (void)enqueueJPEGPixelBufferIfNeeded:(CVPixelBufferRef)pixelBuffer { + if (pixelBuffer == NULL) { + return; + } + + __block BOOL hasListeners = NO; + dispatch_sync(_stateQueue, ^{ + hasListeners = self->_jpegFrameListeners.count > 0; + if (!hasListeners) { + if (self->_pendingJPEGPixelBuffer != nil) { + CVPixelBufferRelease(self->_pendingJPEGPixelBuffer); + self->_pendingJPEGPixelBuffer = nil; + } + self->_jpegEncodeInFlight = NO; + } + }); + if (!hasListeners) { + return; + } + + CVPixelBufferRetain(pixelBuffer); + __block BOOL shouldStartEncode = NO; + dispatch_sync(_stateQueue, ^{ + if (self->_jpegEncodeInFlight) { + if (self->_pendingJPEGPixelBuffer != nil) { + CVPixelBufferRelease(self->_pendingJPEGPixelBuffer); + } + self->_pendingJPEGPixelBuffer = pixelBuffer; + } else { + self->_jpegEncodeInFlight = YES; + shouldStartEncode = YES; + } + }); + + if (shouldStartEncode) { + [self encodeJPEGPixelBuffer:pixelBuffer]; + } +} + +- (void)encodeJPEGPixelBuffer:(CVPixelBufferRef)pixelBuffer { + dispatch_async(_jpegQueue, ^{ + __block NSDictionary *listeners = nil; + __block NSUInteger frameSequence = 0; + dispatch_sync(self->_stateQueue, ^{ + listeners = [self->_jpegFrameListeners copy]; + self->_jpegFrameSequenceValue += 1; + frameSequence = self->_jpegFrameSequenceValue; + }); + + uint64_t timestampUs = XCWCurrentTimestampUs(); + [listeners enumerateKeysAndObjectsUsingBlock:^(__unused NSUUID *token, + XCWPrivateSimulatorJPEGFrameListener *listener, + __unused BOOL *stop) { + CGSize dimensions = CGSizeZero; + NSData *jpegData = XCWJPEGDataFromPixelBuffer(pixelBuffer, + listener.maxEdge, + listener.quality, + &dimensions); + if (jpegData.length > 0) { + listener.handler(jpegData, frameSequence, timestampUs, dimensions); + } + }]; + CVPixelBufferRelease(pixelBuffer); + + __block CVPixelBufferRef nextPixelBuffer = nil; + dispatch_sync(self->_stateQueue, ^{ + nextPixelBuffer = self->_pendingJPEGPixelBuffer; + self->_pendingJPEGPixelBuffer = nil; + if (nextPixelBuffer == nil) { + self->_jpegEncodeInFlight = NO; + } + }); + if (nextPixelBuffer != nil) { + [self encodeJPEGPixelBuffer:nextPixelBuffer]; + } + }); +} + - (void)signalReadinessIfNeededLocked { if (_didSignalReadiness || !_displayReadyValue) { return; diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index bc02dae2..f2971829 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -30,6 +30,16 @@ typedef struct xcw_native_frame { typedef void (*xcw_native_frame_callback)(const xcw_native_frame * _Nonnull frame, void * _Nullable user_data); +typedef struct xcw_native_jpeg_frame { + uint64_t frame_sequence; + uint64_t timestamp_us; + uint32_t width; + uint32_t height; + xcw_native_shared_bytes data; +} xcw_native_jpeg_frame; + +typedef void (*xcw_native_jpeg_frame_callback)(const xcw_native_jpeg_frame * _Nonnull frame, void * _Nullable user_data); + void xcw_native_initialize_app(void); void xcw_native_run_main_loop_slice(double duration_seconds); @@ -83,6 +93,11 @@ bool xcw_native_session_open_app_switcher(void * _Nonnull handle, char * _Nullab bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * _Nullable error_message); bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message); void xcw_native_session_set_frame_callback(void * _Nonnull handle, xcw_native_frame_callback _Nullable callback, void * _Nullable user_data); +void xcw_native_session_set_jpeg_frame_callback(void * _Nonnull handle, + xcw_native_jpeg_frame_callback _Nullable callback, + void * _Nullable user_data, + uint32_t max_edge, + double quality); void xcw_native_free_string(char * _Nullable value); void xcw_native_free_bytes(xcw_native_owned_bytes bytes); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index bdbaa45f..ece3487e 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -803,6 +803,19 @@ void xcw_native_session_set_frame_callback(void *handle, xcw_native_frame_callba } } +void xcw_native_session_set_jpeg_frame_callback(void *handle, + xcw_native_jpeg_frame_callback callback, + void *user_data, + uint32_t max_edge, + double quality) { + @autoreleasepool { + [XCWNativeSessionFromHandle(handle) setJPEGFrameCallback:callback + userData:user_data + maxEdge:max_edge + quality:quality]; + } +} + void xcw_native_free_string(char *value) { if (value != NULL) { free(value); diff --git a/cli/native/XCWNativeSession.h b/cli/native/XCWNativeSession.h index db3fd3fb..baae2ade 100644 --- a/cli/native/XCWNativeSession.h +++ b/cli/native/XCWNativeSession.h @@ -36,6 +36,10 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error; - (void)setFrameCallback:(xcw_native_frame_callback _Nullable)callback userData:(void * _Nullable)userData; +- (void)setJPEGFrameCallback:(xcw_native_jpeg_frame_callback _Nullable)callback + userData:(void * _Nullable)userData + maxEdge:(uint32_t)maxEdge + quality:(double)quality; - (void)disconnect; @end diff --git a/cli/native/XCWNativeSession.m b/cli/native/XCWNativeSession.m index d96c28d2..03ae1c94 100644 --- a/cli/native/XCWNativeSession.m +++ b/cli/native/XCWNativeSession.m @@ -25,8 +25,11 @@ static xcw_native_shared_bytes XCWSharedBytesFromData(NSData *data) { @implementation XCWNativeSession { id _listenerToken; + id _jpegListenerToken; xcw_native_frame_callback _frameCallback; + xcw_native_jpeg_frame_callback _jpegFrameCallback; void *_frameCallbackUserData; + void *_jpegFrameCallbackUserData; } - (nullable instancetype)initWithUDID:(NSString *)udid @@ -194,11 +197,54 @@ - (void)setFrameCallback:(xcw_native_frame_callback)callback }]; } +- (void)setJPEGFrameCallback:(xcw_native_jpeg_frame_callback)callback + userData:(void *)userData + maxEdge:(uint32_t)maxEdge + quality:(double)quality { + _jpegFrameCallback = callback; + _jpegFrameCallbackUserData = userData; + + if (_jpegListenerToken != nil) { + [self.session removeJPEGFrameListener:_jpegListenerToken]; + _jpegListenerToken = nil; + } + + if (callback == NULL) { + return; + } + + __weak typeof(self) weakSelf = self; + _jpegListenerToken = [self.session addJPEGFrameListenerWithMaxEdge:(NSUInteger)maxEdge + quality:quality + handler:^(NSData *jpegData, + NSUInteger frameSequence, + uint64_t timestampUs, + CGSize dimensions) { + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil || strongSelf->_jpegFrameCallback == NULL) { + return; + } + + xcw_native_jpeg_frame frame = { + .frame_sequence = (uint64_t)frameSequence, + .timestamp_us = timestampUs, + .width = (uint32_t)llround(dimensions.width), + .height = (uint32_t)llround(dimensions.height), + .data = XCWSharedBytesFromData(jpegData), + }; + strongSelf->_jpegFrameCallback(&frame, strongSelf->_jpegFrameCallbackUserData); + }]; +} + - (void)disconnect { if (_listenerToken != nil) { [self.session removeEncodedFrameListener:_listenerToken]; _listenerToken = nil; } + if (_jpegListenerToken != nil) { + [self.session removeJPEGFrameListener:_jpegListenerToken]; + _jpegListenerToken = nil; + } [self.session disconnect]; } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 6425be97..1cc14d86 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -51,6 +51,7 @@ import type { StreamEncoder, StreamFps, StreamQualityPreset, + StreamTransport, } from "../features/stream/streamTypes"; import { useLiveStream } from "../features/stream/useLiveStream"; import { DebugPanel } from "../features/toolbar/DebugPanel"; @@ -110,6 +111,8 @@ const REMOTE_STREAM_DEFAULTS: StreamConfig = { fps: 30, quality: "balanced", }; +const MJPEG_DEFAULT_FPS = 30; +const MJPEG_DEFAULT_QUALITY: StreamQualityPreset = "low"; const STREAM_CONFIG_SYNC_INTERVAL_MS = 1500; const STREAM_CONFIG_USER_CHANGE_GRACE_MS = 1000; const STREAM_ENCODER_VALUES = new Set([ @@ -117,6 +120,11 @@ const STREAM_ENCODER_VALUES = new Set([ "hardware", "software", ]); +const STREAM_TRANSPORT_VALUES = new Set([ + "auto", + "mjpeg", + "webrtc", +]); clearLegacyVolatileUiState(); interface StreamQualityResponse { @@ -163,6 +171,43 @@ function shouldUseRemoteStreamDefault(apiRoot: string): boolean { ); } +function readStreamTransportQueryParam(): StreamTransport { + const value = new URLSearchParams(window.location.search).get("stream"); + return value && STREAM_TRANSPORT_VALUES.has(value as StreamTransport) + ? (value as StreamTransport) + : "auto"; +} + +function defaultStreamConfigForTransport( + remote: boolean, + transport: StreamTransport, +): StreamConfig { + const base = remote ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS; + if (transport !== "mjpeg") { + return base; + } + return { + ...base, + fps: MJPEG_DEFAULT_FPS, + maxEdge: undefined, + quality: MJPEG_DEFAULT_QUALITY, + }; +} + +function writeStreamTransportQueryParam(transport: StreamTransport) { + const url = new URL(window.location.href); + if (transport === "auto") { + url.searchParams.delete("stream"); + } else { + url.searchParams.set("stream", transport); + } + window.history.replaceState( + null, + "", + `${url.pathname}${url.search}${url.hash}`, + ); +} + function shouldRenderNativeChrome(simulator: SimulatorMetadata): boolean { const identifier = simulator.deviceTypeIdentifier ?? ""; const name = simulator.name ?? ""; @@ -254,6 +299,9 @@ export function AppShell({ remoteStream = shouldUseRemoteStreamDefault(apiRoot), }: AppShellProps = {}) { configureSimDeckClient({ apiRoot }); + const initialStreamTransportRef = useRef( + readStreamTransportQueryParam(), + ); const [initialUiState] = useState(readPersistedUiState); const [initialSelectedUDID] = useState( () => @@ -336,10 +384,18 @@ export function AppShell({ readStoredFlag(TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, true), ); const [streamConfig, setStreamConfig] = useState(() => - remoteStream ? REMOTE_STREAM_DEFAULTS : LOCAL_STREAM_DEFAULTS, + defaultStreamConfigForTransport( + remoteStream, + initialStreamTransportRef.current, + ), + ); + const [streamTransport, setStreamTransport] = useState( + initialStreamTransportRef.current, ); const [streamConfigApplyKey, setStreamConfigApplyKey] = useState(0); - const [streamConfigReady, setStreamConfigReady] = useState(false); + const [streamConfigReady, setStreamConfigReady] = useState( + initialStreamTransportRef.current === "mjpeg", + ); const [touchIndicators, setTouchIndicators] = useState([]); const menuRef = useRef(null); @@ -358,7 +414,10 @@ export function AppShell({ const accessibilityRequestIdRef = useRef(0); const accessibilityLoadingRef = useRef(false); const streamConfigRequestIdRef = useRef(0); - const streamConfigUserChangeAtRef = useRef(0); + const streamConfigUserChangeAtRef = useRef( + initialStreamTransportRef.current === "mjpeg" ? Date.now() : 0, + ); + const streamConfigUserTouchedRef = useRef(false); const controlSocketRef = useRef<{ udid: string; socket: WebSocket; @@ -504,9 +563,11 @@ export function AppShell({ simulator: selectedSimulator, streamConfig, streamConfigApplyKey, + streamTransport, }); const updateStreamEncoder = useCallback((encoder: StreamEncoder) => { + streamConfigUserTouchedRef.current = true; streamConfigUserChangeAtRef.current = Date.now(); setStreamConfigReady(true); setStreamConfigApplyKey((current) => current + 1); @@ -514,6 +575,7 @@ export function AppShell({ }, []); const updateStreamFps = useCallback((fps: StreamFps) => { + streamConfigUserTouchedRef.current = true; streamConfigUserChangeAtRef.current = Date.now(); setStreamConfigReady(true); setStreamConfigApplyKey((current) => current + 1); @@ -521,12 +583,30 @@ export function AppShell({ }, []); const updateStreamQuality = useCallback((quality: StreamQualityPreset) => { + streamConfigUserTouchedRef.current = true; streamConfigUserChangeAtRef.current = Date.now(); setStreamConfigReady(true); setStreamConfigApplyKey((current) => current + 1); setStreamConfig((current) => ({ ...current, maxEdge: undefined, quality })); }, []); + const updateStreamTransport = useCallback((transport: StreamTransport) => { + setStreamTransport(transport); + writeStreamTransportQueryParam(transport); + if (transport !== "mjpeg" || streamConfigUserTouchedRef.current) { + return; + } + streamConfigUserChangeAtRef.current = Date.now(); + setStreamConfigReady(true); + setStreamConfigApplyKey((current) => current + 1); + setStreamConfig((current) => ({ + ...current, + fps: MJPEG_DEFAULT_FPS, + maxEdge: undefined, + quality: MJPEG_DEFAULT_QUALITY, + })); + }, []); + useEffect(() => { if ( !selectedSimulator || @@ -1580,6 +1660,7 @@ export function AppShell({ onStreamEncoderChange={updateStreamEncoder} onStreamFpsChange={updateStreamFps} onStreamQualityChange={updateStreamQuality} + onStreamTransportChange={updateStreamTransport} onShutdown={() => { if (!selectedSimulator) { return; @@ -1628,6 +1709,7 @@ export function AppShell({ !selectedSimulatorTransitionKind, )} streamConfig={streamConfig} + streamTransport={streamTransport} showStopButton={Boolean( selectedSimulator?.isBooted && !selectedSimulatorTransitionKind, )} diff --git a/client/src/features/simulators/SimulatorMenu.tsx b/client/src/features/simulators/SimulatorMenu.tsx index 174ccbc1..ab5b9207 100644 --- a/client/src/features/simulators/SimulatorMenu.tsx +++ b/client/src/features/simulators/SimulatorMenu.tsx @@ -6,6 +6,7 @@ import type { StreamEncoder, StreamFps, StreamQualityPreset, + StreamTransport, } from "../stream/streamTypes"; import { SimulatorRow } from "./SimulatorRow"; @@ -25,6 +26,7 @@ interface SimulatorMenuProps { onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; onStreamQualityChange: (quality: StreamQualityPreset) => void; + onStreamTransportChange: (transport: StreamTransport) => void; onToggleAppearance: () => void; onToggleDebug: () => void; onToggleMenu: () => void; @@ -34,6 +36,7 @@ interface SimulatorMenuProps { selectedSimulator: SimulatorMetadata | null; setSelectedUDID: (udid: string) => void; streamConfig: StreamConfig; + streamTransport: StreamTransport; touchOverlayVisible: boolean; } @@ -53,6 +56,7 @@ export function SimulatorMenu({ onStreamEncoderChange, onStreamFpsChange, onStreamQualityChange, + onStreamTransportChange, onToggleAppearance, onToggleDebug, onToggleMenu, @@ -62,6 +66,7 @@ export function SimulatorMenu({ selectedSimulator, setSelectedUDID, streamConfig, + streamTransport, touchOverlayVisible, }: SimulatorMenuProps) { const fpsOptions = remoteStream @@ -123,9 +128,21 @@ export function SimulatorMenu({
Stream - {formatStreamConfigSummary(streamConfig)} + {formatStreamConfigSummary(streamConfig, streamTransport)}
+
+ {STREAM_TRANSPORTS.map((option) => ( + + ))} +
{STREAM_ENCODERS.map((option) => (
- {STREAM_QUALITY_OPTIONS.map((option) => ( + {qualityOptions.map((option) => (
-
- {STREAM_TRANSPORTS.map((option) => ( - - ))} -
+
{STREAM_ENCODERS.map((option) => (
-
- {qualityOptions.map((option) => ( - - ))} -
+
@@ -267,8 +293,6 @@ const H264_STREAM_QUALITY_OPTIONS: Array<{ }> = [ { label: "Auto", value: "auto" }, { label: "Full", value: "full" }, - { label: "Max", value: "quality" }, - { label: "Smooth", value: "smooth" }, { label: "1280", value: "balanced" }, { label: "1080", value: "economy" }, { label: "720", value: "low" }, @@ -297,6 +321,27 @@ const MJPEG_QUALITY_LABELS: Partial> = { tiny: "JPEG 0.62", }; +const H264_QUALITY_LABELS: Partial> = { + auto: "Auto", + balanced: "1280px", + economy: "1080px", + full: "Full res", + low: "720px", + quality: "Full+", + smooth: "1170px", + tiny: "540px", +}; + +function streamQualityOptionLabel( + quality: StreamQualityPreset, + transport: StreamTransport, +): string { + if (transport === "mjpeg") { + return MJPEG_QUALITY_LABELS[quality] ?? "JPEG quality"; + } + return H264_QUALITY_LABELS[quality] ?? quality; +} + function MenuIcon() { return ( @@ -317,8 +362,9 @@ function formatStreamConfigSummary( const transportLabel = transport === "h264" ? "H264 WS" : transport.toUpperCase(); const resolution = - typeof streamConfig.maxEdge === "number" && streamConfig.maxEdge > 0 + H264_QUALITY_LABELS[streamConfig.quality] ?? + (typeof streamConfig.maxEdge === "number" && streamConfig.maxEdge > 0 ? `${streamConfig.maxEdge}px` - : "Full res"; + : "Full res"); return `${transportLabel} / ${resolution} / ${streamConfig.fps} fps`; } diff --git a/client/src/features/simulators/useSimulatorList.ts b/client/src/features/simulators/useSimulatorList.ts index a9492742..1fb8abfb 100644 --- a/client/src/features/simulators/useSimulatorList.ts +++ b/client/src/features/simulators/useSimulatorList.ts @@ -3,8 +3,8 @@ import { startTransition, useEffect, useRef, useState } from "react"; import { listSimulators } from "../../api/simulators"; import type { SimulatorMetadata } from "../../api/types"; -const LOCAL_REFRESH_MS = 5000; -const REMOTE_REFRESH_MS = 10000; +const LOCAL_REFRESH_MS = 30000; +const REMOTE_REFRESH_MS = 60000; const REMOTE_ERROR_REFRESH_MS = 15000; const REMOTE_REQUEST_TIMEOUT_MS = 12000; @@ -89,13 +89,27 @@ export function useSimulatorList({ void loadSimulators(cancelled).finally(scheduleNext); }; + const refreshWhenVisible = () => { + if (!cancelled && document.visibilityState === "visible") { + if (timeoutId) { + window.clearTimeout(timeoutId); + timeoutId = 0; + } + run(); + } + }; + run(); + document.addEventListener("visibilitychange", refreshWhenVisible); + window.addEventListener("focus", refreshWhenVisible); return () => { cancelled = true; if (timeoutId) { window.clearTimeout(timeoutId); } + document.removeEventListener("visibilitychange", refreshWhenVisible); + window.removeEventListener("focus", refreshWhenVisible); }; }, [remote]); diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 033e7f3c..514d4abd 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -36,7 +36,6 @@ const H264_WS_HEADER_BYTES = 40; const H264_WS_MAGIC = 0x53444831; const H264_WS_FLAG_KEYFRAME = 1 << 0; const H264_WS_FLAG_CONFIG = 1 << 1; -const H264_WS_MAX_DECODE_QUEUE = 3; const H264_WS_LOCAL_AUTO_PROFILES: StreamQualityPreset[] = [ "low", "economy", @@ -61,6 +60,7 @@ const CONTROL_BACKLOG_DROP_BYTES = 4096; let activeWebRtcControlChannel: RTCDataChannel | null = null; let activeWebRtcTelemetryChannel: RTCDataChannel | null = null; let activeInputSocket: WebSocket | null = null; +let activeH264StreamSocket: WebSocket | null = null; let activeStreamClient: StreamWorkerClient | null = null; export type StreamBackend = "h264-ws" | "mjpeg" | "webrtc"; @@ -75,10 +75,11 @@ export function sendWebRtcControlMessage( ); } -export function sendWebRtcClientStats(stats: unknown): boolean { - return sendDataChannelMessage( - activeWebRtcTelemetryChannel, - JSON.stringify({ stats, type: "clientStats" }), +export function sendStreamClientStats(stats: unknown): boolean { + const encoded = JSON.stringify({ stats, type: "clientStats" }); + return ( + sendDataChannelMessage(activeWebRtcTelemetryChannel, encoded) || + sendWebSocketMessage(activeH264StreamSocket, encoded) ); } @@ -92,6 +93,17 @@ export function sendWebRtcStreamControl(options: { ); } +function sendStreamQualityConfig(config: StreamConfig): boolean { + const encoded = JSON.stringify({ + config: streamQualityPayload(config), + type: "streamQuality", + }); + return ( + sendDataChannelMessage(activeWebRtcControlChannel, encoded) || + sendWebSocketMessage(activeH264StreamSocket, encoded) + ); +} + function sendDataChannelMessage( channel: RTCDataChannel | null, encoded: string, @@ -456,19 +468,6 @@ class MjpegStreamClient implements StreamClientBackend { status: { detail: "Opening MJPEG stream", state: "connecting" }, }); - try { - await postStreamConfigWithAuthRetry(target.streamConfig, { - remote: target.remote, - }); - } catch (error) { - if (!target.remote) { - throw error; - } - } - if (generation !== this.connectGeneration) { - return; - } - const image = document.createElement("img"); image.alt = ""; image.className = "stream-video"; @@ -549,11 +548,6 @@ class MjpegStreamClient implements StreamClientBackend { async applyStreamConfig(config?: StreamConfig) { this.streamConfig = config; - if (config) { - await postStreamConfigWithAuthRetry(config, { - remote: this.streamTarget?.remote, - }); - } const target = this.streamTarget; if (target && this.shouldReconnect) { this.connect({ ...target, streamConfig: config }); @@ -771,22 +765,9 @@ class H264WebSocketStreamClient implements StreamClientBackend { status: { detail: "Opening H264 WebSocket stream", state: "connecting" }, }); - try { - await postStreamConfigWithAuthRetry(effectiveConfig, { - remote: target.remote, - }); - } catch (error) { - if (!target.remote) { - throw error; - } - } - if (generation !== this.connectGeneration) { - return; - } - const socket = new WebSocket( webSocketApiUrl( - `/api/simulators/${encodeURIComponent(target.udid)}/h264`, + `/api/simulators/${encodeURIComponent(target.udid)}/h264${streamQualityQuery(effectiveConfig)}`, ), ); socket.binaryType = "arraybuffer"; @@ -794,6 +775,7 @@ class H264WebSocketStreamClient implements StreamClientBackend { socket.addEventListener("open", () => { if (socket === this.streamSocket) { socket.binaryType = "arraybuffer"; + activeH264StreamSocket = socket; this.startAdaptiveQuality(generation); } }); @@ -813,6 +795,9 @@ class H264WebSocketStreamClient implements StreamClientBackend { this.handleSocketMessage(event.data); }); socket.addEventListener("close", () => { + if (activeH264StreamSocket === socket) { + activeH264StreamSocket = null; + } if (socket === this.streamSocket && this.shouldReconnect) { this.handleError("H264 WebSocket stream closed."); } @@ -839,6 +824,9 @@ class H264WebSocketStreamClient implements StreamClientBackend { this.pendingFrameSequences.clear(); this.h264DecodeTimestampUs = 0; this.streamSocket?.close(); + if (activeH264StreamSocket === this.streamSocket) { + activeH264StreamSocket = null; + } this.streamSocket = null; this.inputSocket?.close(); if (activeInputSocket === this.inputSocket) { @@ -887,15 +875,22 @@ class H264WebSocketStreamClient implements StreamClientBackend { this.autoProfileStableSamples = 0; const effectiveConfig = h264WebSocketStreamConfig(config, this.autoProfile); this.streamConfig = config; + this.clearAdaptiveQuality(); + if (config?.quality === "auto") { + this.startAdaptiveQuality(this.connectGeneration); + } if (effectiveConfig) { - await postStreamConfigWithAuthRetry(effectiveConfig, { - remote: this.streamTarget?.remote, + if (!sendStreamQualityConfig(effectiveConfig)) { + await postStreamConfigWithAuthRetry(effectiveConfig, { + remote: this.streamTarget?.remote, + }); + } + this.sendControl({ + forceKeyframe: true, + snapshot: true, + type: "streamControl", }); } - const target = this.streamTarget; - if (target && this.shouldReconnect) { - this.connect({ ...target, streamConfig: config }); - } } private connectInputSocket(target: StreamConnectTarget, generation: number) { @@ -952,25 +947,6 @@ class H264WebSocketStreamClient implements StreamClientBackend { return; } this.stats.decodeQueueSize = decoder.decodeQueueSize; - if ( - decoder.decodeQueueSize > H264_WS_MAX_DECODE_QUEUE && - !frame.isKeyFrame - ) { - this.stats.droppedFrames += 1; - this.stats.decoderDroppedFrames += 1; - this.closePendingFrame(); - this.closeDecoder(); - this.pendingFrameSequences.clear(); - this.waitingForKeyFrame = true; - this.stats.waitingForKeyFrame = true; - this.onMessage({ type: "stats", stats: { ...this.stats } }); - this.sendControl({ - forceKeyframe: true, - frameSequence: frame.sequence, - type: "streamControl", - }); - return; - } const { EncodedVideoChunk } = webCodecsConstructors(); if (!EncodedVideoChunk) { @@ -1406,11 +1382,13 @@ class H264WebSocketStreamClient implements StreamClientBackend { if (!effectiveConfig) { return; } - await postStreamConfigWithAuthRetry(effectiveConfig, { - remote: this.streamTarget?.remote, - }).catch(() => { - // Stream-quality adaptation is opportunistic; the stream socket handles reachability. - }); + if (!sendStreamQualityConfig(effectiveConfig)) { + await postStreamConfigWithAuthRetry(effectiveConfig, { + remote: this.streamTarget?.remote, + }).catch(() => { + // Stream-quality adaptation is opportunistic; the stream socket handles reachability. + }); + } this.sendControl({ forceKeyframe: true, type: "streamControl" }); } @@ -1616,18 +1594,6 @@ class WebRtcStreamClient implements StreamClientBackend { }); try { - try { - await postStreamConfigWithAuthRetry(target.streamConfig, { - remote: target.remote, - }); - } catch (error) { - if (!target.remote) { - throw error; - } - } - if (generation !== this.connectGeneration) { - return; - } const health = await fetchHealth().catch(() => null); if (generation !== this.connectGeneration) { return; @@ -1805,13 +1771,10 @@ class WebRtcStreamClient implements StreamClientBackend { return; } const generation = ++this.streamConfigGeneration; - await postStreamConfigWithAuthRetry(config, { remote: this.remoteMode }); - if (generation !== this.streamConfigGeneration) { - return; + if (!sendStreamQualityConfig(config)) { + await postStreamConfigWithAuthRetry(config, { remote: this.remoteMode }); } - const target = this.streamTarget; - if (target && this.shouldReconnect) { - await this.connect({ ...target, streamConfig: config }); + if (generation !== this.streamConfigGeneration) { return; } this.sendControl({ forceKeyframe: true, type: "streamControl" }); @@ -2283,20 +2246,9 @@ class WebRtcStreamClient implements StreamClientBackend { url: window.location.href, userAgent: window.navigator.userAgent, }; - if (sendWebRtcClientStats(payload) || this.remoteMode) { + if (sendStreamClientStats(payload) || this.remoteMode) { return; } - void fetch( - new URL(apiUrl("/api/client-stream-stats"), window.location.href), - { - body: JSON.stringify(payload), - cache: "no-store", - headers: apiHeaders(), - method: "POST", - }, - ).catch(() => { - // Diagnostics only. - }); } private drawVideoFrame = () => { @@ -2573,16 +2525,36 @@ async function postStreamConfigWithAuthRetry( function postStreamConfig(config: StreamConfig): Promise { return fetch(apiUrl("/api/stream-quality"), { - body: JSON.stringify({ - fps: config.fps, - profile: config.quality === "auto" ? "economy" : config.quality, - videoCodec: config.encoder, - }), + body: JSON.stringify(streamQualityPayload(config)), headers: apiHeaders(), method: "POST", }); } +function streamQualityPayload(config: StreamConfig): { + fps: number; + profile: StreamQualityPreset; + videoCodec: string; +} { + return { + fps: config.fps, + profile: config.quality === "auto" ? "economy" : config.quality, + videoCodec: config.encoder, + }; +} + +function streamQualityQuery(config: StreamConfig | undefined): string { + if (!config) { + return ""; + } + const params = new URLSearchParams(); + const payload = streamQualityPayload(config); + params.set("fps", String(payload.fps)); + params.set("profile", payload.profile); + params.set("videoCodec", payload.videoCodec); + return `?${params.toString()}`; +} + function postWebRtcOffer( target: StreamConnectTarget, localDescription: RTCSessionDescription, @@ -2593,6 +2565,9 @@ function postWebRtcOffer( body: JSON.stringify({ clientId: target.clientId, sdp: localDescription.sdp, + streamConfig: target.streamConfig + ? streamQualityPayload(target.streamConfig) + : undefined, type: localDescription.type, }), headers: apiHeaders(), diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index fdae4b55..7b6d90d4 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -7,7 +7,7 @@ import type { Size } from "../viewport/types"; import { createEmptyStreamStats } from "./stats"; import { buildStreamTarget, - sendWebRtcClientStats, + sendStreamClientStats, StreamWorkerClient, type StreamBackend, type VisualArtifactSample, @@ -25,7 +25,7 @@ const FPS_SAMPLE_INTERVAL_MS = 1000; const CLIENT_TELEMETRY_INTERVAL_MS = 1000; const REMOTE_CLIENT_TELEMETRY_INTERVAL_MS = 5000; const CLIENT_TELEMETRY_ID_STORAGE_KEY = "simdeck.streamClientId"; -const VISUAL_ARTIFACT_TELEMETRY_INTERVAL_MS = 1000; +const VISUAL_ARTIFACT_TELEMETRY_INTERVAL_MS = 30000; interface UseLiveStreamOptions { canvasElement: HTMLCanvasElement | null; @@ -368,7 +368,13 @@ export function useLiveStream({ visualSampleCount: latestVisualArtifactSampleCountRef.current, visibilityState: document.visibilityState, }; - if (sendWebRtcClientStats(payload) || remote) { + if ( + sendStreamClientStats(payload) || + remote || + streamTransport === "h264" || + streamTransport === "webrtc" || + streamTransport === "auto" + ) { return; } void fetch(buildClientTelemetryUrl(), { @@ -389,7 +395,7 @@ export function useLiveStream({ return () => { window.clearInterval(intervalId); }; - }, [remote, simulator?.udid]); + }, [remote, simulator?.udid, streamTransport]); return { deviceNaturalSize, diff --git a/client/src/styles/components.css b/client/src/styles/components.css index c1f26dca..184b3a80 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -357,6 +357,40 @@ grid-template-columns: repeat(2, minmax(0, 1fr)); } +.menu-field { + display: grid; + gap: 5px; +} + +.menu-field-label { + color: var(--text-muted); + font-size: 10px; + font-weight: 600; + letter-spacing: 0; + text-transform: uppercase; +} + +.menu-select { + width: 100%; + height: 32px; + padding: 0 28px 0 10px; + border: 1px solid var(--border-subtle); + border-radius: 6px; + background: var(--surface); + color: var(--text); + font: inherit; + font-size: 12px; +} + +.menu-select:hover { + background: var(--surface-hover); +} + +.menu-select:focus-visible { + outline: 2px solid color-mix(in srgb, var(--accent) 45%, transparent); + outline-offset: 2px; +} + .menu-option { min-width: 0; height: 30px; diff --git a/docs/api/health.md b/docs/api/health.md index 924c7571..aa8eaf84 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -117,7 +117,7 @@ from hardware to software encoding. ### Client stream stats -`client_streams` is a rolling buffer of the most recent reports a client posted to `POST /api/client-stream-stats`. The server keeps the last 48 entries per `(clientId, kind)` pair. +`client_streams` is a rolling buffer of the most recent reports from browser stream transports. WebRTC clients normally send reports over the telemetry data channel, H.264 WebSocket clients send them on the stream socket, and simple clients can still post to `POST /api/client-stream-stats`. The server keeps the last 48 entries per `(clientId, kind)` pair. The browser client uses these to render its in-app diagnostics overlay and to size its decoder workers. Every field is optional except `clientId` and `kind`; see [`ClientStreamStats`](https://github.com/NativeScript/SimDeck/blob/main/server/src/metrics/counters.rs) for the full schema. diff --git a/docs/api/rest.md b/docs/api/rest.md index 899292f9..a0070e24 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -76,24 +76,25 @@ Returns the active stream encoder settings and available quality profiles. ### `POST /api/stream-quality` -Updates the active stream encoder settings for newly encoded frames. The browser -UI uses this before WebRTC negotiation when the user selects encoder, FPS, or -quality. +Updates the active stream encoder settings for newly encoded frames. Browser +clients normally send these updates on the active WebRTC data channel or H.264 +WebSocket; this endpoint remains for scripts and fallback clients. ```json { "videoCodec": "hardware", - "fps": 120, - "profile": "quality" + "fps": 60, + "profile": "full" } ``` `videoCodec` accepts `hardware` or `software` from the UI, and the API also accepts `auto`. `fps` is clamped to the local stream range. Browser viewers show -six profiles: `full` (4096 px at 60 fps), `quality` (4096 px high bitrate), -`balanced` (1280 px), `economy` (1080 px), `low` (720 px), and `tiny` (540 px). -The API still accepts the legacy `fast`, `smooth`, and `ci-software` profiles -for CLI/provider compatibility. When +five H.264 resolution profiles: `full` (4096 px at 60 fps), `balanced` +(1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), +and `tiny` (540 px at 30 fps). The API still accepts the legacy `quality`, +`fast`, `smooth`, and `ci-software` profiles for CLI/provider compatibility. +When `profile` is provided, its resolution preset is applied; send `maxEdge` without `profile` for a custom resolution cap. @@ -165,6 +166,11 @@ and the server responds with an SDP answer for a receive-only H.264 video track: ```json { "sdp": "v=0\r\n...", + "streamConfig": { + "fps": 60, + "profile": "full", + "videoCodec": "auto" + }, "type": "offer" } ``` @@ -179,15 +185,23 @@ and the server responds with an SDP answer for a receive-only H.264 video track: The endpoint requires the active simulator stream to produce H.264-compatible samples. The bundled browser client always uses this endpoint. -The browser also opens a `simdeck-control` data channel. In addition to input -messages, clients can tune the stream attached to that peer: +The browser also opens `simdeck-control` and `simdeck-telemetry` data channels. +In addition to input messages, clients can request a keyframe or tune the +stream attached to that peer: ```json -{ "type": "streamControl", "profile": "thumb" } +{ "type": "streamControl", "forceKeyframe": true } ``` -Supported profiles are `thumb`/`thumbnail`, `focus`/`full`, and `paused`. -Clients may also send `fps`, `forceKeyframe`, or `snapshot` fields. +```json +{ "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } +``` + +The telemetry channel accepts: + +```json +{ "type": "clientStats", "stats": { "clientId": "browser", "kind": "webrtc" } } +``` ### `GET /api/simulators/{udid}/h264` @@ -209,12 +223,26 @@ layout: | 36 | 4 | H.264 sample byte length, big-endian | The optional decoder config bytes follow the header, then the encoded H.264 -sample bytes. Clients can send text `streamControl` messages on this socket, -for example `{ "type": "streamControl", "forceKeyframe": true }`. Touch and -keyboard input should use the separate `/api/simulators/{udid}/input` -WebSocket. The video socket is latest-frame oriented: clients should drop stale -decoded frames locally and request a keyframe if the decoder loses sync, rather -than ACKing every rendered frame. +sample bytes. Clients can pass initial stream settings as query parameters +(`profile`, `fps`, `videoCodec`) and can send text control messages on the same +socket: + +```json +{ "type": "streamControl", "forceKeyframe": true } +``` + +```json +{ "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } +``` + +```json +{ "type": "clientStats", "stats": { "clientId": "browser", "kind": "page" } } +``` + +Touch and keyboard input should use the separate `/api/simulators/{udid}/input` +WebSocket. The video socket is latest-frame oriented: clients should paint the +latest decoded frame locally and request a keyframe if the decoder loses sync, +rather than ACKing every rendered frame. ### `POST /api/simulators/{udid}/open-url` diff --git a/docs/cli/flags.md b/docs/cli/flags.md index a11c35de..ecfad846 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -36,7 +36,7 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas | `--client-root` | bundled `client/dist` | Override the static browser client directory. | | `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | -| `--stream-quality` | `smooth` | Realtime stream quality profile: `full`, `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | +| `--stream-quality` | `full` | Realtime stream quality profile: `full`, `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | | `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 6e912fe0..20111a52 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -29,7 +29,7 @@ Key modules: | `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | | `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | | `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | -| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted via `/api/client-stream-stats`. | +| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | | `server/src/logs.rs` | `os_log` log streaming and filtering. | The Rust server runs the tokio runtime on a worker thread while the AppKit main loop spins on the main thread. The native bridge needs the main loop to deliver display callbacks and HID events. diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index 77ee4a24..9bb94502 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -62,7 +62,7 @@ This starts or reuses the project daemon, serves the bundled browser client, and | `--client-root` | bundled `client/dist` | Override the static browser client directory. | | `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). | | `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | -| `--stream-quality` | `smooth` | Realtime stream quality profile, including `full`, `low`, `tiny`, and `ci-software`. | +| `--stream-quality` | `full` | Realtime stream quality profile, including `full`, `low`, `tiny`, and `ci-software`. | | `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | | `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | diff --git a/docs/guide/video.md b/docs/guide/video.md index 9145b683..3afb7797 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -32,7 +32,7 @@ It is CLI-only because it is meant for less capable machines where freshness matters more than maximum smoothness. The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`. -The browser UI exposes stream controls for encoder, FPS, and quality. H264 modes include `full` (4096 px at 60 fps), `quality` (4096 px high bitrate), `balanced` (1280 px), `economy` (1080 px), `low` (720 px), and `tiny` (540 px). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality. +The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality. ## Remote WebRTC ICE @@ -74,7 +74,7 @@ H264 WS uses the same native H.264 encoder as WebRTC, but sends each encoded sample on a binary WebSocket at: ```http -GET /api/simulators/{udid}/h264 +GET /api/simulators/{udid}/h264?profile=full&fps=60&videoCodec=auto ``` Each message starts with a compact SimDeck header, followed by optional AVC @@ -82,10 +82,12 @@ decoder config and the encoded sample bytes. The browser decodes with WebCodecs, keeps only the latest decoded frame, and paints on `requestAnimationFrame` so stale frames do not build latency. Input stays on the separate `/api/simulators/{udid}/input` WebSocket so large video frames do -not block touch and keyboard messages. H264 WS defaults to the `full` profile -on loopback and `auto` quality for remote sessions. H264 `Auto` starts at -`full` on loopback; remote `Auto` starts lower but can climb through `smooth`, -`balanced`, and `full` after sustained low decode/render pressure. +not block touch and keyboard messages. Stream-quality updates and client stats +use the WebRTC data channel or the H264 WS video socket instead of repeatedly +posting REST requests. H264 WS defaults to the `full` profile on loopback and +`auto` quality for remote sessions. H264 `Auto` starts at `full` on loopback; +remote `Auto` starts lower but can climb through the internal `smooth` step +(1170 px), `balanced`, and `full` after sustained low decode/render pressure. MJPEG uses the private display bridge directly, encodes the latest `CVPixelBuffer` as JPEG in native code, and serves it as: @@ -143,7 +145,7 @@ A few practical guidelines: - **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Pass `--video-codec software` only when the shared hardware encoder is unavailable or performs worse on that host. - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. -- **Studio providers default to software H.264 plus `--stream-quality smooth`.** This profile uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. +- **Studio providers default to software H.264 plus `--stream-quality smooth`.** `smooth` is an internal/provider profile, not a browser picker item. It uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. - **Use `Auto` for the default MJPEG stream.** It encodes the native frame size at JPEG quality `0.70`, targets 30 fps, lowers JPEG quality when encoded frames are too large or the HTTP stream backs up, and raises it again after sustained low pressure. MJPEG does not apply the H.264 `maxEdge` caps unless a caller explicitly passes `maxEdge` to the raw MJPEG endpoint. - **The remote browser renders WebRTC as a native `` stream.** The canvas remains for input geometry and diagnostics, and fallback mode keeps simulator controls on the WebSocket input channel. - **Use `--stream-quality ci-software` for denser virtualized CI Macs.** This profile uses software H.264 at a 960-pixel longest edge, targets 24 fps, lowers bitrate pressure, and favors fresh frames over full-resolution sharpness. @@ -185,7 +187,9 @@ multiple frames in a row exceeded the budget. For hardware H.264 this usually means the shared VideoToolbox encoder is saturated; lower resolution/FPS or switch to software H.264. -Clients can also push their decoder/renderer stats back to the server: +Clients can also push their decoder/renderer stats back to the server. Browser +clients normally send these over the WebRTC telemetry data channel or the H264 +WS stream socket; REST remains available for simple clients: ```http POST /api/client-stream-stats diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 776ff9cd..9281ecd1 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -140,16 +140,34 @@ struct TouchSequencePayload { events: Vec, } -#[derive(Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] -struct StreamQualityPayload { - profile: Option, +pub(crate) struct StreamQualityPayload { + pub(crate) profile: Option, #[serde(rename = "videoCodec")] - video_codec: Option, - max_edge: Option, - fps: Option, - min_bitrate: Option, - bits_per_pixel: Option, + pub(crate) video_codec: Option, + pub(crate) max_edge: Option, + pub(crate) fps: Option, + pub(crate) min_bitrate: Option, + pub(crate) bits_per_pixel: Option, +} + +impl StreamQualityPayload { + fn has_any_value(&self) -> bool { + self.profile + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()) + || self + .video_codec + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()) + || self.max_edge.is_some() + || self.fps.is_some() + || self.min_bitrate.is_some() + || self.bits_per_pixel.is_some() + } } #[derive(Clone, Copy)] @@ -233,30 +251,29 @@ const STREAM_QUALITY_PROFILES: &[StreamQualityProfile] = &[ id: "economy", label: "Economy", max_edge: 1080, - fps: 24, - min_bitrate: 1_500_000, - bits_per_pixel: 3, + fps: 30, + min_bitrate: 3_500_000, + bits_per_pixel: 6, }, StreamQualityProfile { id: "low", label: "Low", max_edge: 720, fps: 30, - min_bitrate: 900_000, - bits_per_pixel: 2, + min_bitrate: 2_000_000, + bits_per_pixel: 5, }, StreamQualityProfile { id: "tiny", label: "Tiny", max_edge: 540, - fps: 24, - min_bitrate: 600_000, - bits_per_pixel: 2, + fps: 30, + min_bitrate: 1_200_000, + bits_per_pixel: 4, }, ]; -const VISIBLE_STREAM_QUALITY_PROFILE_IDS: &[&str] = - &["full", "quality", "balanced", "economy", "low", "tiny"]; +const VISIBLE_STREAM_QUALITY_PROFILE_IDS: &[&str] = &["full", "balanced", "economy", "low", "tiny"]; static STREAM_CONFIG_LOCK: OnceLock> = OnceLock::new(); @@ -716,6 +733,16 @@ async fn set_stream_quality( State(state): State, Json(payload): Json, ) -> Result, AppError> { + apply_stream_quality_payload(&state, &payload).map(json) +} + +pub(crate) fn apply_stream_quality_payload( + state: &AppState, + payload: &StreamQualityPayload, +) -> Result { + if !payload.has_any_value() { + return Ok(stream_quality_response(&state.config)); + } let video_codec = payload .video_codec .as_deref() @@ -749,7 +776,7 @@ async fn set_stream_quality( && current.profile == next_profile && current.video_codec == next_video_codec { - return Ok(json(json_value!(stream_quality_response(&state.config)))); + return Ok(stream_quality_response(&state.config)); } env::set_var("SIMDECK_REALTIME_MAX_EDGE", limits.max_edge.to_string()); @@ -773,7 +800,7 @@ async fn set_stream_quality( } state.registry.reconfigure_video_encoders(); - Ok(json(json_value!(stream_quality_response(&state.config)))) + Ok(stream_quality_response(&state.config)) } fn stream_quality_response(config: &Config) -> Value { @@ -1340,9 +1367,10 @@ async fn control_socket( async fn h264_socket( State(state): State, Path(udid): Path, + Query(query): Query, websocket: WebSocketUpgrade, ) -> impl IntoResponse { - websocket.on_upgrade(move |socket| handle_h264_socket(state, udid, socket)) + websocket.on_upgrade(move |socket| handle_h264_socket(state, udid, query, socket)) } async fn mjpeg_stream( @@ -1549,7 +1577,17 @@ fn mjpeg_frame_part(frame: &crate::transport::packet::JpegFramePacket) -> Bytes part.freeze() } -async fn handle_h264_socket(state: AppState, udid: String, socket: WebSocket) { +async fn handle_h264_socket( + state: AppState, + udid: String, + initial_quality: StreamQualityPayload, + socket: WebSocket, +) { + if initial_quality.has_any_value() { + if let Err(error) = apply_stream_quality_payload(&state, &initial_quality) { + tracing::debug!("Failed to apply H264 WebSocket stream quality for {udid}: {error}"); + } + } let session = match state.registry.get_or_create_async(&udid).await { Ok(session) => session, Err(error) => { @@ -1600,7 +1638,7 @@ async fn handle_h264_socket(state: AppState, udid: String, socket: WebSocket) { break; } }; - if !handle_h264_socket_control_message(&session, &message) { + if !handle_h264_socket_message(&state, &session, &message) { break; } } @@ -1647,7 +1685,8 @@ async fn handle_h264_socket(state: AppState, udid: String, socket: WebSocket) { } } -fn handle_h264_socket_control_message( +fn handle_h264_socket_message( + state: &AppState, session: &SimulatorSession, message: &Message, ) -> bool { @@ -1660,30 +1699,53 @@ fn handle_h264_socket_control_message( Message::Close(_) => return false, Message::Ping(_) | Message::Pong(_) => return true, }; - let Ok(value) = serde_json::from_str::(text) else { + let Ok(message) = serde_json::from_str::(text) else { return true; }; - match value.get("type").and_then(Value::as_str) { - Some("streamControl") => {} - _ => return true, - } - if value - .get("forceKeyframe") - .and_then(Value::as_bool) - .unwrap_or(false) - { - session.request_keyframe(); - } - if value - .get("snapshot") - .and_then(Value::as_bool) - .unwrap_or(false) - { - session.request_refresh(); + match message { + H264SocketMessage::ClientStats { stats } => { + if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { + state.metrics.record_client_stream_stats(*stats); + } + } + H264SocketMessage::StreamControl { + force_keyframe, + snapshot, + } => { + if force_keyframe.unwrap_or(false) { + session.request_keyframe(); + } + if snapshot.unwrap_or(false) { + session.request_refresh(); + } + } + H264SocketMessage::StreamQuality { config } => { + if let Err(error) = apply_stream_quality_payload(state, &config) { + tracing::debug!("Failed to apply H264 WebSocket stream quality: {error}"); + } else { + session.request_keyframe(); + } + } } true } +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum H264SocketMessage { + ClientStats { + stats: Box, + }, + StreamControl { + #[serde(rename = "forceKeyframe")] + force_keyframe: Option, + snapshot: Option, + }, + StreamQuality { + config: StreamQualityPayload, + }, +} + fn h264_ws_frame_is_supported(frame: &FramePacket) -> bool { frame .codec @@ -4295,5 +4357,4 @@ mod tests { assert_eq!(&message[40..44], b"avcc"); assert_eq!(&message[44..], b"h264-sample"); } - } diff --git a/server/src/main.rs b/server/src/main.rs index 4868f727..7549d455 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -701,7 +701,7 @@ struct StreamQualityEnvironment { bits_per_pixel: u32, } -const DEFAULT_LOCAL_STREAM_QUALITY_PROFILE: &str = "quality"; +const DEFAULT_LOCAL_STREAM_QUALITY_PROFILE: &str = "full"; fn local_stream_quality_profile( low_latency: bool, @@ -752,23 +752,23 @@ fn stream_quality_env_for_profile(profile: &str) -> anyhow::Result Ok(StreamQualityEnvironment { profile: "economy", max_edge: 1080, - fps: 24, - min_bitrate: 1_500_000, - bits_per_pixel: 3, + fps: 30, + min_bitrate: 3_500_000, + bits_per_pixel: 6, }), "low" => Ok(StreamQualityEnvironment { profile: "low", max_edge: 720, fps: 30, - min_bitrate: 900_000, - bits_per_pixel: 2, + min_bitrate: 2_000_000, + bits_per_pixel: 5, }), "tiny" => Ok(StreamQualityEnvironment { profile: "tiny", max_edge: 540, - fps: 24, - min_bitrate: 600_000, - bits_per_pixel: 2, + fps: 30, + min_bitrate: 1_200_000, + bits_per_pixel: 4, }), "ci-software" => Ok(StreamQualityEnvironment { profile: "ci-software", diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 3da66160..793ac8a4 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -1,4 +1,7 @@ -use crate::api::routes::{run_control_message, AppState, ControlMessage}; +use crate::api::routes::{ + apply_stream_quality_payload, run_control_message, AppState, ControlMessage, + StreamQualityPayload, +}; use crate::error::AppError; use crate::metrics::counters::ClientStreamStats; use bytes::Bytes; @@ -63,6 +66,8 @@ struct WebRtcMediaStreamToken { pub struct WebRtcOfferPayload { pub client_id: Option, pub sdp: String, + #[serde(rename = "streamConfig")] + pub stream_config: Option, #[serde(rename = "type")] pub kind: String, pub transport: Option, @@ -107,6 +112,9 @@ pub async fn create_answer( "WebRTC preview supports media tracks only.", )); } + if let Some(stream_config) = payload.stream_config.as_ref() { + apply_stream_quality_payload(&state, stream_config)?; + } info!( "WebRTC offer for {udid}: remote_candidates={} remote_candidate_types={} ice_servers={} ice_transport_policy={}", count_sdp_candidates(&payload.sdp), @@ -422,6 +430,13 @@ fn attach_control_data_channel( } let _ = stream_control_tx.send(command); } + WebRtcDataChannelMessage::StreamQuality { config } => { + if let Err(error) = apply_stream_quality_payload(&state, &config) { + warn!("WebRTC stream quality update failed for {udid}: {error}"); + } else { + session.request_keyframe(); + } + } } return; } @@ -500,6 +515,9 @@ enum WebRtcDataChannelMessage { force_keyframe: Option, snapshot: Option, }, + StreamQuality { + config: StreamQualityPayload, + }, } #[derive(Clone, Debug)]