diff --git a/AGENTS.md b/AGENTS.md index d8e616e2..7e26c2da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,6 +70,7 @@ Private simulator behavior is implemented locally in: - Accessibility bridge: `cli/XCWAccessibilityBridge.*` The current repo uses the private boot path, private display bridge, and private accessibility translation bridge directly. The browser streams frames from that bridge, injects touch and keyboard events through the same native session layer, inspects accessibility through `AccessibilityPlatformTranslation`, and renders device chrome from `cli/XCWChromeRenderer.*`. +Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, and mute buttons dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. ## Build and Run diff --git a/README.md b/README.md index 380626a9..9d544db7 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,8 @@ simdeck key-combo --modifiers cmd --key a simdeck type "hello" simdeck type --file message.txt simdeck button lock --duration-ms 1000 +simdeck button volume-up +simdeck button action --duration-ms 1000 simdeck batch --step "tap --label Continue" --step "type 'hello'" --step "wait-for --label hello" simdeck dismiss-keyboard simdeck home diff --git a/cli/DFPrivateSimulatorDisplayBridge.h b/cli/DFPrivateSimulatorDisplayBridge.h index c4bce914..5c99a66c 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.h +++ b/cli/DFPrivateSimulatorDisplayBridge.h @@ -12,6 +12,14 @@ typedef NS_ENUM(NSInteger, DFPrivateSimulatorTouchPhase) { DFPrivateSimulatorTouchPhaseCancelled = 3, } NS_SWIFT_NAME(PrivateSimulatorTouchPhase); +typedef NS_ENUM(NSInteger, DFPrivateSimulatorTouchEdge) { + DFPrivateSimulatorTouchEdgeNone = 0, + DFPrivateSimulatorTouchEdgeLeft = 1, + DFPrivateSimulatorTouchEdgeTop = 2, + DFPrivateSimulatorTouchEdgeBottom = 3, + DFPrivateSimulatorTouchEdgeRight = 4, +} NS_SWIFT_NAME(PrivateSimulatorTouchEdge); + NS_SWIFT_NAME(PrivateSimulatorDisplayBridgeDelegate) @protocol DFPrivateSimulatorDisplayBridgeDelegate @@ -47,6 +55,12 @@ NS_SWIFT_NAME(PrivateSimulatorDisplayBridge) phase:(DFPrivateSimulatorTouchPhase)phase error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(sendTouch(normalizedX:normalizedY:phase:)); +- (BOOL)sendEdgeTouchAtNormalizedX:(double)normalizedX + normalizedY:(double)normalizedY + phase:(DFPrivateSimulatorTouchPhase)phase + edge:(DFPrivateSimulatorTouchEdge)edge + error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(sendEdgeTouch(normalizedX:normalizedY:phase:edge:)); + - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1 normalizedY1:(double)normalizedY1 normalizedX2:(double)normalizedX2 @@ -66,6 +80,11 @@ NS_SWIFT_NAME(PrivateSimulatorDisplayBridge) - (BOOL)pressHardwareButtonNamed:(NSString *)buttonName durationMs:(NSUInteger)durationMs error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(pressHardwareButton(named:durationMs:)); +- (BOOL)sendHardwareButtonNamed:(NSString *)buttonName + pressed:(BOOL)pressed + usagePage:(nullable NSNumber *)usagePage + usage:(nullable NSNumber *)usage + error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(sendHardwareButton(named:pressed:usagePage:usage:)); - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(rotateRight()); - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(rotateLeft()); diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 437e653b..1c90a467 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -33,6 +33,7 @@ typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, NSEventType type, NSSize displaySize, IndigoHIDEdge edge); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEvent9Fn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, uint32_t direction, double unused1, double unused2, double widthPoints, double heightPoints); +typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventEdgeFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, IndigoHIDEdge edge, double widthPoints, double heightPoints); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardArbitraryFn)(int keyCode, int op); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target); @@ -99,6 +100,11 @@ static const uint32_t DFIndigoMouseDirectionDown = 1; static const uint32_t DFIndigoMouseDirectionMove = 0; static const uint32_t DFIndigoMouseDirectionUp = 2; +static const IndigoHIDEdge DFIndigoHIDEdgeNone = 0; +static const IndigoHIDEdge DFIndigoHIDEdgeLeft = 1; +static const IndigoHIDEdge DFIndigoHIDEdgeTop = 2; +static const IndigoHIDEdge DFIndigoHIDEdgeBottom = 3; +static const IndigoHIDEdge DFIndigoHIDEdgeRight = 4; static const int DFKeyboardDirectionDown = 1; static const int DFKeyboardDirectionUp = 2; static const uint32_t DFButtonDirectionDown = 1; @@ -1136,6 +1142,53 @@ static uint32_t DFIndigoMouseDirectionForPhase(DFPrivateSimulatorTouchPhase phas displaySize.height); } +static IndigoHIDEdge DFIndigoHIDEdgeForPrivateTouchEdge(DFPrivateSimulatorTouchEdge edge) { + switch (edge) { + case DFPrivateSimulatorTouchEdgeLeft: + return DFIndigoHIDEdgeLeft; + case DFPrivateSimulatorTouchEdgeTop: + return DFIndigoHIDEdgeTop; + case DFPrivateSimulatorTouchEdgeBottom: + return DFIndigoHIDEdgeBottom; + case DFPrivateSimulatorTouchEdgeRight: + return DFIndigoHIDEdgeRight; + case DFPrivateSimulatorTouchEdgeNone: + default: + return DFIndigoHIDEdgeNone; + } +} + +static IndigoHIDMessage *DFCreateIndigoEdgeTouchMessage(CGPoint normalizedPoint, DFPrivateSimulatorTouchPhase phase, DFPrivateSimulatorTouchEdge edge) { + DFIndigoHIDMessageForMouseNSEventEdgeFn mouseMessage = (DFIndigoHIDMessageForMouseNSEventEdgeFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); + if (mouseMessage == NULL) { + return NULL; + } + + CGPoint ratioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint.x)), + fmax(0.0, fmin(1.0, normalizedPoint.y)) + ); + return mouseMessage(&ratioPoint, + NULL, + DFIndigoTouchTarget, + DFIndigoMouseEventTypeForPhase(phase), + DFIndigoHIDEdgeForPrivateTouchEdge(edge), + 1.0, + 1.0); +} + +static IndigoHIDMessage *DFCreateIndigoEdgeTouchMessageOnMain(CGPoint normalizedPoint, DFPrivateSimulatorTouchPhase phase, DFPrivateSimulatorTouchEdge edge) { + if ([NSThread isMainThread]) { + return DFCreateIndigoEdgeTouchMessage(normalizedPoint, phase, edge); + } + + __block IndigoHIDMessage *message = NULL; + dispatch_sync(dispatch_get_main_queue(), ^{ + message = DFCreateIndigoEdgeTouchMessage(normalizedPoint, phase, edge); + }); + return message; +} + static void DFWarmIndigoHIDServices(id hidClient) { if (hidClient == nil) { return; @@ -3286,8 +3339,66 @@ - (BOOL)pressHardwareButtonNamed:(NSString *)buttonName } return [self pressHomeButton:error]; } + if ([normalizedName isEqualToString:@"app-switcher"]) { + if (durationMs > 0) { + [NSThread sleepForTimeInterval:(NSTimeInterval)durationMs / 1000.0]; + } + return [self openAppSwitcher:error]; + } + + NSUInteger holdMs = durationMs > 0 ? durationMs : 100; + if (![self sendHardwareButtonNamed:buttonName pressed:YES usagePage:nil usage:nil error:error]) { + return NO; + } + [NSThread sleepForTimeInterval:(NSTimeInterval)holdMs / 1000.0]; + return [self sendHardwareButtonNamed:buttonName pressed:NO usagePage:nil usage:nil error:error]; +} + +- (BOOL)sendHardwareButtonNamed:(NSString *)buttonName + pressed:(BOOL)pressed + usagePage:(NSNumber *)usagePage + usage:(NSNumber *)usage + error:(NSError * _Nullable __autoreleasing *)error { + NSString *normalizedName = buttonName.lowercaseString ?: @""; + uint32_t operation = pressed ? DFButtonDirectionDown : DFButtonDirectionUp; + + if ([normalizedName isEqualToString:@"home"]) { + __block BOOL success = NO; + __block NSError *dispatchError = nil; + dispatch_block_t work = ^{ + if (self->_hidClient == nil) { + dispatchError = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit did not provide a headless HID client for Home." + ); + return; + } + const DFHomeButtonHIDStrategy strategy = { + "IndigoHIDMessageForHIDArbitrary page=0x0c usage=0x40 (Menu) target=0x32", + NO, + 0, + DFConsumerControlUsagePage, + 0x40, + DFIndigoTouchTarget + }; + success = DFSendHomeStrategyEdge(self->_hidClient, &strategy, operation, &dispatchError); + }; + if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) { + work(); + } else { + dispatch_sync(_callbackQueue, work); + } + if (!success && error != NULL) { + *error = dispatchError ?: DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + [NSString stringWithFormat:@"SimulatorKit rejected Home button %@.", pressed ? @"down" : @"up"] + ); + } + return success; + } static NSDictionary *buttonCodes; + static NSDictionary *> *arbitraryButtons; static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ buttonCodes = @{ @@ -3298,10 +3409,19 @@ - (BOOL)pressHardwareButtonNamed:(NSString *)buttonName @"side": @4, @"siri": @5, }; + arbitraryButtons = @{ + @"power": @[ @(DFConsumerControlUsagePage), @48 ], + @"volume-up": @[ @(DFConsumerControlUsagePage), @233 ], + @"volume-down": @[ @(DFConsumerControlUsagePage), @234 ], + @"action": @[ @(0x0b), @45 ], + @"mute": @[ @(0x0b), @46 ], + }; }); NSNumber *buttonCode = buttonCodes[normalizedName]; - if (buttonCode == nil) { + NSArray *arbitraryUsage = arbitraryButtons[normalizedName]; + BOOL hasExplicitUsage = usagePage != nil && usage != nil; + if (buttonCode == nil && arbitraryUsage == nil && !hasExplicitUsage) { if (error != NULL) { *error = DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @@ -3323,36 +3443,36 @@ - (BOOL)pressHardwareButtonNamed:(NSString *)buttonName return; } - uint32_t code = buttonCode.unsignedIntValue; - uint32_t targets[] = { DFIndigoTouchTarget, 0x2 }; - for (NSUInteger targetIndex = 0; targetIndex < sizeof(targets) / sizeof(targets[0]); targetIndex++) { - NSError *downError = nil; - IndigoHIDMessage *down = DFCreateButtonMessage(code, DFButtonDirectionDown, targets[targetIndex], &downError); - if (down == NULL) { - dispatchError = downError; - continue; - } - if (!DFSendHIDMessage(self->_hidClient, down, YES, &downError)) { - dispatchError = downError; - continue; + if (hasExplicitUsage || arbitraryUsage.count >= 2) { + uint32_t page = hasExplicitUsage ? usagePage.unsignedIntValue : arbitraryUsage[0].unsignedIntValue; + uint32_t usageValue = hasExplicitUsage ? usage.unsignedIntValue : arbitraryUsage[1].unsignedIntValue; + NSError *messageError = nil; + IndigoHIDMessage *message = DFCreateArbitraryHIDMessage(DFIndigoTouchTarget, page, usageValue, operation, &messageError); + if (message == NULL || !DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) { + dispatchError = messageError; + return; } - if (durationMs > 0) { - [NSThread sleepForTimeInterval:(NSTimeInterval)durationMs / 1000.0]; - } + DFLog(@"Sending arbitrary HID button `%@` page=0x%x usage=0x%x operation=%u", buttonName, page, usageValue, operation); + success = YES; + return; + } - NSError *upError = nil; - IndigoHIDMessage *up = DFCreateButtonMessage(code, DFButtonDirectionUp, targets[targetIndex], &upError); - if (up == NULL) { - dispatchError = upError; + uint32_t code = buttonCode.unsignedIntValue; + uint32_t targets[] = { DFIndigoTouchTarget, 0x2 }; + for (NSUInteger targetIndex = 0; targetIndex < sizeof(targets) / sizeof(targets[0]); targetIndex++) { + NSError *messageError = nil; + IndigoHIDMessage *message = DFCreateButtonMessage(code, operation, targets[targetIndex], &messageError); + if (message == NULL) { + dispatchError = messageError; continue; } - if (!DFSendHIDMessage(self->_hidClient, up, YES, &upError)) { - dispatchError = upError; + if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) { + dispatchError = messageError; continue; } - DFLog(@"Sending hardware button `%@` code=%u target=0x%x durationMs=%lu", buttonName, code, targets[targetIndex], (unsigned long)durationMs); + DFLog(@"Sending hardware button `%@` code=%u target=0x%x operation=%u", buttonName, code, targets[targetIndex], operation); success = YES; return; } @@ -3367,7 +3487,7 @@ - (BOOL)pressHardwareButtonNamed:(NSString *)buttonName if (!success && error != NULL) { *error = dispatchError ?: DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, - [NSString stringWithFormat:@"SimulatorKit rejected hardware button `%@`.", buttonName ?: @""] + [NSString stringWithFormat:@"SimulatorKit rejected hardware button `%@` %@.", buttonName ?: @"", pressed ? @"down" : @"up"] ); } @@ -3563,6 +3683,80 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX return success; } +- (BOOL)sendEdgeTouchAtNormalizedX:(double)normalizedX + normalizedY:(double)normalizedY + phase:(DFPrivateSimulatorTouchPhase)phase + edge:(DFPrivateSimulatorTouchEdge)edge + error:(NSError * _Nullable __autoreleasing *)error { + __block BOOL success = NO; + __block NSError *dispatchError = nil; + + dispatch_block_t work = ^{ + if (self->_hidClient == nil) { + dispatchError = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit did not provide a headless HID client for this simulator." + ); + return; + } + + CGFloat clampedX = (CGFloat)fmax(0.0, fmin(1.0, normalizedX)); + CGFloat clampedY = (CGFloat)fmax(0.0, fmin(1.0, normalizedY)); + CGSize displaySize = self->_displayPixelSize; + if (displaySize.width < 1.0 || displaySize.height < 1.0) { + displaySize = CGSizeMake(1.0, 1.0); + } + CGPoint point = CGPointMake( + clampedX * fmax(displaySize.width - 1.0, 1.0), + clampedY * fmax(displaySize.height - 1.0, 1.0) + ); + + IndigoHIDMessage *message = DFCreateIndigoEdgeTouchMessageOnMain(CGPointMake(clampedX, clampedY), phase, edge); + if (message == NULL) { + dispatchError = DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit failed to create an edge-aware Indigo HID touch packet." + ); + return; + } + + NSError *messageError = nil; + if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) { + dispatchError = messageError ?: DFMakeError( + DFPrivateSimulatorErrorCodeTouchDispatchFailed, + @"SimulatorKit rejected the edge-aware Indigo HID touch packet." + ); + return; + } + + if (DFVerboseTouchLoggingEnabled() && phase != DFPrivateSimulatorTouchPhaseMoved) { + DFLog(@"Sending edge Indigo HID touch edge=%ld at pixel (%.1f, %.1f) ratio (%.4f, %.4f)", + (long)edge, point.x, point.y, clampedX, clampedY); + } + + self->_lastTouchPoint = point; + self->_hasLastTouchPoint = YES; + if (phase == DFPrivateSimulatorTouchPhaseEnded || phase == DFPrivateSimulatorTouchPhaseCancelled) { + self->_lastTouchPoint = CGPointZero; + self->_hasLastTouchPoint = NO; + } + + success = YES; + }; + + if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) { + work(); + } else { + dispatch_sync(_callbackQueue, work); + } + + if (!success && error != NULL) { + *error = dispatchError; + } + + return success; +} + - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1 normalizedY1:(double)normalizedY1 normalizedX2:(double)normalizedX2 diff --git a/cli/XCWChromeRenderer.h b/cli/XCWChromeRenderer.h index d6abd048..64f7c513 100644 --- a/cli/XCWChromeRenderer.h +++ b/cli/XCWChromeRenderer.h @@ -6,6 +6,13 @@ NS_ASSUME_NONNULL_BEGIN + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; ++ (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName + includeButtons:(BOOL)includeButtons + error:(NSError * _Nullable * _Nullable)error; ++ (nullable NSData *)buttonPNGDataForDeviceName:(NSString *)deviceName + buttonName:(NSString *)buttonName + pressed:(BOOL)pressed + error:(NSError * _Nullable * _Nullable)error; + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; + (nullable NSDictionary *)profileForDeviceName:(NSString *)deviceName diff --git a/cli/XCWChromeRenderer.m b/cli/XCWChromeRenderer.m index fc5b0faf..0926c2a8 100644 --- a/cli/XCWChromeRenderer.m +++ b/cli/XCWChromeRenderer.m @@ -4,6 +4,16 @@ #import static NSString * const XCWChromeRendererErrorDomain = @"SimDeck.ChromeRenderer"; + +@interface XCWChromeRenderer () ++ (NSArray *> *)buttonProfilesForChromeInfo:(NSDictionary *)chromeInfo + chromeSize:(CGSize)chromeSize + chromeOffset:(CGPoint)chromeOffset; ++ (nullable NSDictionary *)inputNamed:(NSString *)buttonName + chromeInfo:(NSDictionary *)chromeInfo + error:(NSError * _Nullable __autoreleasing *)error; +@end + @implementation XCWChromeRenderer + (nullable NSDictionary *)profileForDeviceName:(NSString *)deviceName @@ -17,6 +27,12 @@ @implementation XCWChromeRenderer + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable __autoreleasing *)error { + return [self PNGDataForDeviceName:deviceName includeButtons:YES error:error]; +} + ++ (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName + includeButtons:(BOOL)includeButtons + error:(NSError * _Nullable __autoreleasing *)error { NSDictionary *chromeInfo = [self chromeInfoForDeviceName:deviceName error:error]; if (chromeInfo == nil) { return nil; @@ -36,7 +52,7 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName [self numberValue:profile[@"totalHeight"]]); CGFloat chromeX = [self numberValue:profile[@"chromeX"]]; CGFloat chromeY = [self numberValue:profile[@"chromeY"]]; - BOOL drawNonTopInputsBeforeBody = YES; + BOOL drawNonTopInputsBeforeBody = includeButtons; CGFloat scale = 3.0; NSInteger pixelWidth = MAX((NSInteger)ceil(renderSize.width * scale), 1); @@ -94,7 +110,7 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName CGContextRelease(context); return nil; } - if (!drawNonTopInputsBeforeBody) { + if (includeButtons && !drawNonTopInputsBeforeBody) { if (![self drawInputImagesForChromeInfo:chromeInfo inSize:chromeSize context:context @@ -117,14 +133,16 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName return nil; } CGContextTranslateCTM(context, chromeX, chromeY); - if (![self drawInputImagesForChromeInfo:chromeInfo - inSize:chromeSize - context:context - onlyOnTop:YES - error:error]) { - CGContextRestoreGState(context); - CGContextRelease(context); - return nil; + if (includeButtons) { + if (![self drawInputImagesForChromeInfo:chromeInfo + inSize:chromeSize + context:context + onlyOnTop:YES + error:error]) { + CGContextRestoreGState(context); + CGContextRelease(context); + return nil; + } } CGContextRestoreGState(context); @@ -177,6 +195,37 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName return data; } ++ (nullable NSData *)buttonPNGDataForDeviceName:(NSString *)deviceName + buttonName:(NSString *)buttonName + pressed:(BOOL)pressed + error:(NSError * _Nullable __autoreleasing *)error { + NSDictionary *chromeInfo = [self chromeInfoForDeviceName:deviceName error:error]; + if (chromeInfo == nil) { + return nil; + } + NSDictionary *input = [self inputNamed:buttonName chromeInfo:chromeInfo error:error]; + if (input == nil) { + return nil; + } + + NSString *assetName = pressed && [input[@"imageDown"] isKindOfClass:[NSString class]] + ? input[@"imageDown"] + : input[@"image"]; + if (assetName.length == 0) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:14 + userInfo:@{ + NSLocalizedDescriptionKey: [NSString stringWithFormat:@"The chrome button `%@` did not specify a renderable image.", buttonName ?: @""], + }]; + } + return nil; + } + + NSString *assetPath = [self resolvedChromeAssetPathForName:assetName chromePath:chromeInfo[@"chromePath"]]; + return [self PNGDataForPDFAtPath:assetPath scale:3.0 error:error]; +} + + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable __autoreleasing *)error { NSDictionary *chromeInfo = [self chromeInfoForDeviceName:deviceName error:error]; @@ -278,6 +327,9 @@ + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName CGFloat chromeX = -CGRectGetMinX(fullFrame); CGFloat chromeY = -CGRectGetMinY(fullFrame); BOOL hasScreenMask = !phoneProfile && [self screenMaskPathForChromeInfo:chromeInfo].length > 0; + NSArray *> *buttons = [self buttonProfilesForChromeInfo:chromeInfo + chromeSize:compositeSize + chromeOffset:CGPointMake(chromeX, chromeY)]; return @{ @"totalWidth": @(CGRectGetWidth(fullFrame)), @@ -293,6 +345,7 @@ + (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName @"cornerRadius": @(cornerRadius), @"chromeCornerRadius": @(chromeCornerRadius), @"hasScreenMask": @(hasScreenMask), + @"buttons": buttons, }; } @@ -636,7 +689,14 @@ + (CGRect)fullFrameForChromeInfo:(NSDictionary *)chromeInfo if (assetSize.width <= 0.0 || assetSize.height <= 0.0) { continue; } - bounds = CGRectUnion(bounds, [self inputFrameForInput:input assetSize:assetSize inSize:chromeSize]); + bounds = CGRectUnion(bounds, [self inputFrameForInput:input + assetSize:assetSize + inSize:chromeSize + offsetName:@"normal"]); + bounds = CGRectUnion(bounds, [self inputFrameForInput:input + assetSize:assetSize + inSize:chromeSize + offsetName:@"rollover"]); } return CGRectIntegral(bounds); } @@ -644,14 +704,24 @@ + (CGRect)fullFrameForChromeInfo:(NSDictionary *)chromeInfo + (CGRect)inputFrameForInput:(NSDictionary *)input assetSize:(CGSize)assetSize inSize:(CGSize)size { + return [self inputFrameForInput:input assetSize:assetSize inSize:size offsetName:@"normal"]; +} + ++ (CGRect)inputFrameForInput:(NSDictionary *)input + assetSize:(CGSize)assetSize + inSize:(CGSize)size + offsetName:(NSString *)offsetName { NSDictionary *offsets = [input[@"offsets"] isKindOfClass:[NSDictionary class]] ? input[@"offsets"] : @{}; - NSDictionary *normalOffset = [offsets[@"normal"] isKindOfClass:[NSDictionary class]] ? offsets[@"normal"] : @{}; - CGFloat offsetX = [self numberValue:normalOffset[@"x"]]; - CGFloat offsetY = [self numberValue:normalOffset[@"y"]]; + NSDictionary *requestedOffset = [offsets[offsetName] isKindOfClass:[NSDictionary class]] ? offsets[offsetName] : nil; + NSDictionary *normalOffset = [offsets[@"normal"] isKindOfClass:[NSDictionary class]] ? offsets[@"normal"] : nil; + NSDictionary *rolloverOffset = [offsets[@"rollover"] isKindOfClass:[NSDictionary class]] ? offsets[@"rollover"] : nil; + NSDictionary *offset = requestedOffset ?: rolloverOffset ?: normalOffset ?: @{}; + CGFloat offsetX = [self numberValue:offset[@"x"]]; + CGFloat offsetY = [self numberValue:offset[@"y"]]; NSString *anchor = [input[@"anchor"] isKindOfClass:[NSString class]] ? input[@"anchor"] : @""; NSString *align = [input[@"align"] isKindOfClass:[NSString class]] ? input[@"align"] : @""; - CGFloat x = offsetX; + CGFloat x = offsetX - (assetSize.width / 2.0); CGFloat y = offsetY; if ([anchor isEqualToString:@"left"]) { x = offsetX - (assetSize.width / 2.0); @@ -670,10 +740,17 @@ + (CGRect)inputFrameForInput:(NSDictionary *)input y = size.height - assetSize.height + offsetY; } } else if ([anchor isEqualToString:@"top"] || [anchor isEqualToString:@"bottom"]) { + CGFloat baseX = 0.0; if ([align isEqualToString:@"center"]) { - x = (size.width - assetSize.width) / 2.0 + offsetX; + baseX = size.width / 2.0; } else if ([align isEqualToString:@"trailing"]) { - x = size.width - assetSize.width + offsetX; + baseX = size.width; + } + x = baseX + offsetX - (assetSize.width / 2.0); + if ([align isEqualToString:@"center"]) { + x = (size.width / 2.0) + offsetX - (assetSize.width / 2.0); + } else if ([align isEqualToString:@"trailing"]) { + x = size.width + offsetX - (assetSize.width / 2.0); } } else if ([align isEqualToString:@"center"]) { x = (size.width - assetSize.width) / 2.0 + offsetX; @@ -684,6 +761,117 @@ + (CGRect)inputFrameForInput:(NSDictionary *)input return CGRectMake(x, y, assetSize.width, assetSize.height); } ++ (NSArray *> *)buttonProfilesForChromeInfo:(NSDictionary *)chromeInfo + chromeSize:(CGSize)chromeSize + chromeOffset:(CGPoint)chromeOffset { + NSDictionary *json = chromeInfo[@"json"]; + NSString *chromePath = chromeInfo[@"chromePath"]; + NSArray *inputs = [json[@"inputs"] isKindOfClass:[NSArray class]] ? json[@"inputs"] : @[]; + NSMutableArray *> *buttons = [NSMutableArray arrayWithCapacity:inputs.count]; + + for (id inputValue in inputs) { + if (![inputValue isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *input = inputValue; + NSString *name = [input[@"name"] isKindOfClass:[NSString class]] ? input[@"name"] : @""; + NSString *assetName = [input[@"image"] isKindOfClass:[NSString class]] ? input[@"image"] : @""; + if (name.length == 0 || assetName.length == 0) { + continue; + } + + NSString *assetPath = [self resolvedChromeAssetPathForName:assetName chromePath:chromePath]; + CGSize assetSize = [self PDFPageSizeAtPath:assetPath]; + if (assetSize.width <= 0.0 || assetSize.height <= 0.0) { + continue; + } + + CGRect rect = [self inputFrameForInput:input assetSize:assetSize inSize:chromeSize]; + rect = CGRectOffset(rect, chromeOffset.x, chromeOffset.y); + if (CGRectGetWidth(rect) <= 0.0 || CGRectGetHeight(rect) <= 0.0) { + continue; + } + + NSDictionary *offsets = [input[@"offsets"] isKindOfClass:[NSDictionary class]] ? input[@"offsets"] : @{}; + NSDictionary *normalOffset = [offsets[@"normal"] isKindOfClass:[NSDictionary class]] ? offsets[@"normal"] : @{}; + NSDictionary *rolloverOffset = [offsets[@"rollover"] isKindOfClass:[NSDictionary class]] ? offsets[@"rollover"] : normalOffset; + NSString *label = [input[@"accessibilityTitle"] isKindOfClass:[NSString class]] ? input[@"accessibilityTitle"] : name; + NSString *type = [input[@"type"] isKindOfClass:[NSString class]] ? input[@"type"] : @""; + NSString *anchor = [input[@"anchor"] isKindOfClass:[NSString class]] ? input[@"anchor"] : @""; + NSString *align = [input[@"align"] isKindOfClass:[NSString class]] ? input[@"align"] : @""; + NSString *imageDownName = [input[@"imageDown"] isKindOfClass:[NSString class]] ? input[@"imageDown"] : @""; + NSString *imageDownDrawMode = [input[@"imageDownDrawMode"] isKindOfClass:[NSString class]] ? input[@"imageDownDrawMode"] : @""; + BOOL onTop = [input[@"onTop"] respondsToSelector:@selector(boolValue)] && [input[@"onTop"] boolValue]; + + NSMutableDictionary *button = [@{ + @"name": name, + @"label": label, + @"type": type, + @"imageName": assetName, + @"x": @(CGRectGetMinX(rect)), + @"y": @(CGRectGetMinY(rect)), + @"width": @(CGRectGetWidth(rect)), + @"height": @(CGRectGetHeight(rect)), + @"anchor": anchor, + @"align": align, + @"onTop": @(onTop), + @"normalOffset": @{ + @"x": @([self numberValue:normalOffset[@"x"]]), + @"y": @([self numberValue:normalOffset[@"y"]]), + }, + @"rolloverOffset": @{ + @"x": @([self numberValue:rolloverOffset[@"x"]]), + @"y": @([self numberValue:rolloverOffset[@"y"]]), + }, + } mutableCopy]; + + if (imageDownName.length > 0) { + button[@"imageDownName"] = imageDownName; + } + if (imageDownDrawMode.length > 0) { + button[@"imageDownDrawMode"] = imageDownDrawMode; + } + + if ([input[@"usagePage"] respondsToSelector:@selector(unsignedIntValue)]) { + button[@"usagePage"] = @([input[@"usagePage"] unsignedIntValue]); + } + if ([input[@"usage"] respondsToSelector:@selector(unsignedIntValue)]) { + button[@"usage"] = @([input[@"usage"] unsignedIntValue]); + } + + [buttons addObject:button]; + } + + return buttons; +} + ++ (nullable NSDictionary *)inputNamed:(NSString *)buttonName + chromeInfo:(NSDictionary *)chromeInfo + error:(NSError * _Nullable __autoreleasing *)error { + NSString *normalizedName = [[buttonName ?: @"" stringByDeletingPathExtension] lowercaseString]; + NSDictionary *json = chromeInfo[@"json"]; + NSArray *inputs = [json[@"inputs"] isKindOfClass:[NSArray class]] ? json[@"inputs"] : @[]; + for (id inputValue in inputs) { + if (![inputValue isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *input = inputValue; + NSString *name = [input[@"name"] isKindOfClass:[NSString class]] ? input[@"name"] : @""; + if ([name.lowercaseString isEqualToString:normalizedName]) { + return input; + } + } + + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:15 + userInfo:@{ + NSLocalizedDescriptionKey: [NSString stringWithFormat:@"The device chrome did not expose a button named `%@`.", buttonName ?: @""], + }]; + } + return nil; +} + + (CGSize)screenSizeForChromeInfo:(NSDictionary *)chromeInfo chromeSize:(CGSize)chromeSize screenScale:(CGFloat)screenScale { diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 33397ceb..f5cfbfa9 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -1103,8 +1103,7 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix int32_t sourceWidth = (int32_t)CVPixelBufferGetWidth(pixelBuffer); int32_t sourceHeight = (int32_t)CVPixelBufferGetHeight(pixelBuffer); OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); - BOOL shouldCopyStableRealtimeBuffer = _realtimeStreamMode || _lowLatencyMode; - if (sourceWidth == targetWidth && sourceHeight == targetHeight && !shouldCopyStableRealtimeBuffer) { + if (sourceWidth == targetWidth && sourceHeight == targetHeight) { CVPixelBufferRetain(pixelBuffer); return pixelBuffer; } diff --git a/cli/XCWPrivateSimulatorSession.h b/cli/XCWPrivateSimulatorSession.h index 90dc69af..a12eab63 100644 --- a/cli/XCWPrivateSimulatorSession.h +++ b/cli/XCWPrivateSimulatorSession.h @@ -40,6 +40,12 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData, phase:(NSString *)phase error:(NSError * _Nullable * _Nullable)error; +- (BOOL)sendEdgeTouchWithNormalizedX:(double)normalizedX + normalizedY:(double)normalizedY + phase:(NSString *)phase + edge:(NSInteger)edge + error:(NSError * _Nullable * _Nullable)error; + - (BOOL)sendMultiTouchWithNormalizedX1:(double)normalizedX1 normalizedY1:(double)normalizedY1 normalizedX2:(double)normalizedX2 @@ -52,6 +58,14 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData, error:(NSError * _Nullable * _Nullable)error; - (BOOL)pressHomeButton:(NSError * _Nullable * _Nullable)error; +- (BOOL)pressHardwareButtonNamed:(NSString *)buttonName + durationMs:(NSUInteger)durationMs + error:(NSError * _Nullable * _Nullable)error; +- (BOOL)sendHardwareButtonNamed:(NSString *)buttonName + pressed:(BOOL)pressed + usagePage:(nullable NSNumber *)usagePage + usage:(nullable NSNumber *)usage + error:(NSError * _Nullable * _Nullable)error; - (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m index 18e9eac4..e5bd5136 100644 --- a/cli/XCWPrivateSimulatorSession.m +++ b/cli/XCWPrivateSimulatorSession.m @@ -253,6 +253,42 @@ - (BOOL)sendTouchWithNormalizedX:(double)normalizedX return [_displayBridge sendTouchAtNormalizedX:normalizedX normalizedY:normalizedY phase:touchPhase error:error]; } +- (BOOL)sendEdgeTouchWithNormalizedX:(double)normalizedX + normalizedY:(double)normalizedY + phase:(NSString *)phase + edge:(NSInteger)edge + error:(NSError * _Nullable __autoreleasing *)error { + DFPrivateSimulatorTouchPhase touchPhase = DFPrivateSimulatorTouchPhaseMoved; + if (![self touchPhaseFromString:phase outPhase:&touchPhase error:error]) { + return NO; + } + + DFPrivateSimulatorTouchEdge touchEdge = DFPrivateSimulatorTouchEdgeNone; + switch (edge) { + case DFPrivateSimulatorTouchEdgeLeft: + touchEdge = DFPrivateSimulatorTouchEdgeLeft; + break; + case DFPrivateSimulatorTouchEdgeTop: + touchEdge = DFPrivateSimulatorTouchEdgeTop; + break; + case DFPrivateSimulatorTouchEdgeBottom: + touchEdge = DFPrivateSimulatorTouchEdgeBottom; + break; + case DFPrivateSimulatorTouchEdgeRight: + touchEdge = DFPrivateSimulatorTouchEdgeRight; + break; + default: + touchEdge = DFPrivateSimulatorTouchEdgeNone; + break; + } + + return [_displayBridge sendEdgeTouchAtNormalizedX:normalizedX + normalizedY:normalizedY + phase:touchPhase + edge:touchEdge + error:error]; +} + - (BOOL)sendMultiTouchWithNormalizedX1:(double)normalizedX1 normalizedY1:(double)normalizedY1 normalizedX2:(double)normalizedX2 @@ -307,6 +343,24 @@ - (BOOL)pressHomeButton:(NSError * _Nullable __autoreleasing *)error { return [_displayBridge pressHomeButton:error]; } +- (BOOL)pressHardwareButtonNamed:(NSString *)buttonName + durationMs:(NSUInteger)durationMs + error:(NSError * _Nullable __autoreleasing *)error { + return [_displayBridge pressHardwareButtonNamed:buttonName durationMs:durationMs error:error]; +} + +- (BOOL)sendHardwareButtonNamed:(NSString *)buttonName + pressed:(BOOL)pressed + usagePage:(NSNumber *)usagePage + usage:(NSNumber *)usage + error:(NSError * _Nullable __autoreleasing *)error { + return [_displayBridge sendHardwareButtonNamed:buttonName + pressed:pressed + usagePage:usagePage + usage:usage + error:error]; +} + - (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error { return [_displayBridge openAppSwitcher:error]; } diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index bc02dae2..a7d81d61 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -40,7 +40,8 @@ bool xcw_native_toggle_appearance(const char * _Nonnull udid, char * _Nullable * bool xcw_native_open_url(const char * _Nonnull udid, const char * _Nonnull url, char * _Nullable * _Nullable error_message); bool xcw_native_launch_bundle(const char * _Nonnull udid, const char * _Nonnull bundle_id, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_get_chrome_profile(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); -xcw_native_owned_bytes xcw_native_render_chrome_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_render_chrome_png(const char * _Nonnull udid, bool include_buttons, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_render_chrome_button_png(const char * _Nonnull udid, const char * _Nonnull button_name, bool pressed, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_recent_logs(const char * _Nonnull udid, double seconds, size_t limit, char * _Nullable * _Nullable error_message); @@ -51,6 +52,7 @@ bool xcw_native_send_key_event(const char * _Nonnull udid, uint16_t key_code, bo bool xcw_native_press_home(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_open_app_switcher(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_press_button(const char * _Nonnull udid, const char * _Nonnull button_name, uint32_t duration_ms, char * _Nullable * _Nullable error_message); +bool xcw_native_send_button(const char * _Nonnull udid, const char * _Nonnull button_name, bool pressed, bool has_usage, uint32_t usage_page, uint32_t usage, char * _Nullable * _Nullable error_message); bool xcw_native_rotate_right(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_rotate_left(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); bool xcw_native_erase_simulator(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); @@ -76,9 +78,12 @@ void xcw_native_session_reconfigure_video_encoder(void * _Nonnull handle); char * _Nullable xcw_native_session_video_encoder_stats(void * _Nonnull handle, char * _Nullable * _Nullable error_message); int32_t xcw_native_session_rotation_quarter_turns(void * _Nonnull handle); bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message); +bool xcw_native_session_send_edge_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, uint32_t edge, char * _Nullable * _Nullable error_message); bool xcw_native_session_send_multitouch(void * _Nonnull handle, double x1, double y1, double x2, double y2, const char * _Nonnull phase, char * _Nullable * _Nullable error_message); bool xcw_native_session_send_key(void * _Nonnull handle, uint16_t key_code, uint32_t modifiers, char * _Nullable * _Nullable error_message); bool xcw_native_session_press_home(void * _Nonnull handle, char * _Nullable * _Nullable error_message); +bool xcw_native_session_press_button(void * _Nonnull handle, const char * _Nonnull button_name, uint32_t duration_ms, char * _Nullable * _Nullable error_message); +bool xcw_native_session_send_button(void * _Nonnull handle, const char * _Nonnull button_name, bool pressed, bool has_usage, uint32_t usage_page, uint32_t usage, char * _Nullable * _Nullable error_message); bool xcw_native_session_open_app_switcher(void * _Nonnull handle, char * _Nullable * _Nullable error_message); bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * _Nullable error_message); bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index bdbaa45f..3fc2376a 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -181,7 +181,7 @@ bool xcw_native_launch_bundle(const char *udid, const char *bundle_id, char **er } } -xcw_native_owned_bytes xcw_native_render_chrome_png(const char *udid, char **error_message) { +xcw_native_owned_bytes xcw_native_render_chrome_png(const char *udid, bool include_buttons, char **error_message) { @autoreleasepool { NSDictionary *simulator = XCWSimulatorRecordForUDID(udid, error_message); if (simulator == nil) { @@ -191,6 +191,7 @@ xcw_native_owned_bytes xcw_native_render_chrome_png(const char *udid, char **err NSError *renderError = nil; NSString *deviceName = simulator[@"deviceTypeName"] ?: simulator[@"name"] ?: @""; NSData *pngData = [XCWChromeRenderer PNGDataForDeviceName:deviceName + includeButtons:include_buttons error:&renderError]; if (pngData == nil) { XCWSetErrorMessage(error_message, renderError); @@ -201,6 +202,28 @@ xcw_native_owned_bytes xcw_native_render_chrome_png(const char *udid, char **err } } +xcw_native_owned_bytes xcw_native_render_chrome_button_png(const char *udid, const char *button_name, bool pressed, char **error_message) { + @autoreleasepool { + NSDictionary *simulator = XCWSimulatorRecordForUDID(udid, error_message); + if (simulator == nil) { + return (xcw_native_owned_bytes){0}; + } + + NSError *renderError = nil; + NSString *deviceName = simulator[@"deviceTypeName"] ?: simulator[@"name"] ?: @""; + NSData *pngData = [XCWChromeRenderer buttonPNGDataForDeviceName:deviceName + buttonName:XCWStringFromCString(button_name) + pressed:pressed + error:&renderError]; + if (pngData == nil) { + XCWSetErrorMessage(error_message, renderError); + return (xcw_native_owned_bytes){0}; + } + + return XCWOwnedBytesFromData(pngData); + } +} + xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char *udid, char **error_message) { @autoreleasepool { NSDictionary *simulator = XCWSimulatorRecordForUDID(udid, error_message); @@ -536,6 +559,26 @@ bool xcw_native_press_button(const char *udid, const char *button_name, uint32_t } } +bool xcw_native_send_button(const char *udid, const char *button_name, bool pressed, bool has_usage, uint32_t usage_page, uint32_t usage, char **error_message) { + @autoreleasepool { + DFPrivateSimulatorDisplayBridge *bridge = XCWInputBridgeForUDID(udid, error_message); + if (bridge == nil) { + return false; + } + NSError *error = nil; + BOOL ok = [bridge sendHardwareButtonNamed:XCWStringFromCString(button_name) + pressed:pressed + usagePage:has_usage ? @(usage_page) : nil + usage:has_usage ? @(usage) : nil + error:&error]; + [bridge disconnect]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + bool xcw_native_rotate_right(const char *udid, char **error_message) { @autoreleasepool { DFPrivateSimulatorDisplayBridge *bridge = XCWInputBridgeForUDID(udid, error_message); @@ -724,6 +767,21 @@ bool xcw_native_session_send_touch(void *handle, double x, double y, const char } } +bool xcw_native_session_send_edge_touch(void *handle, double x, double y, const char *phase, uint32_t edge, char **error_message) { + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeSessionFromHandle(handle) sendEdgeTouchAtX:x + y:y + phase:XCWStringFromCString(phase) + edge:(NSInteger)edge + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + bool xcw_native_session_send_multitouch(void *handle, double x1, double y1, double x2, double y2, const char *phase, char **error_message) { @autoreleasepool { NSError *error = nil; @@ -764,6 +822,34 @@ bool xcw_native_session_press_home(void *handle, char **error_message) { } } +bool xcw_native_session_press_button(void *handle, const char *button_name, uint32_t duration_ms, char **error_message) { + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeSessionFromHandle(handle) pressHardwareButtonNamed:XCWStringFromCString(button_name) + durationMs:(NSUInteger)duration_ms + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + +bool xcw_native_session_send_button(void *handle, const char *button_name, bool pressed, bool has_usage, uint32_t usage_page, uint32_t usage, char **error_message) { + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeSessionFromHandle(handle) sendHardwareButtonNamed:XCWStringFromCString(button_name) + pressed:pressed + usagePage:has_usage ? @(usage_page) : nil + usage:has_usage ? @(usage) : nil + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + bool xcw_native_session_open_app_switcher(void *handle, char **error_message) { @autoreleasepool { NSError *error = nil; diff --git a/cli/native/XCWNativeSession.h b/cli/native/XCWNativeSession.h index db3fd3fb..0018af53 100644 --- a/cli/native/XCWNativeSession.h +++ b/cli/native/XCWNativeSession.h @@ -21,6 +21,11 @@ NS_ASSUME_NONNULL_BEGIN y:(double)y phase:(NSString *)phase error:(NSError * _Nullable * _Nullable)error; +- (BOOL)sendEdgeTouchAtX:(double)x + y:(double)y + phase:(NSString *)phase + edge:(NSInteger)edge + error:(NSError * _Nullable * _Nullable)error; - (BOOL)sendMultiTouchAtX1:(double)x1 y1:(double)y1 x2:(double)x2 @@ -31,6 +36,14 @@ NS_ASSUME_NONNULL_BEGIN modifiers:(uint32_t)modifiers error:(NSError * _Nullable * _Nullable)error; - (BOOL)pressHome:(NSError * _Nullable * _Nullable)error; +- (BOOL)pressHardwareButtonNamed:(NSString *)buttonName + durationMs:(NSUInteger)durationMs + error:(NSError * _Nullable * _Nullable)error; +- (BOOL)sendHardwareButtonNamed:(NSString *)buttonName + pressed:(BOOL)pressed + usagePage:(nullable NSNumber *)usagePage + usage:(nullable NSNumber *)usage + error:(NSError * _Nullable * _Nullable)error; - (BOOL)openAppSwitcher:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateRight:(NSError * _Nullable * _Nullable)error; - (BOOL)rotateLeft:(NSError * _Nullable * _Nullable)error; diff --git a/cli/native/XCWNativeSession.m b/cli/native/XCWNativeSession.m index d96c28d2..9140729b 100644 --- a/cli/native/XCWNativeSession.m +++ b/cli/native/XCWNativeSession.m @@ -117,6 +117,18 @@ - (BOOL)sendTouchAtX:(double)x return [self.session sendTouchWithNormalizedX:x normalizedY:y phase:phase error:error]; } +- (BOOL)sendEdgeTouchAtX:(double)x + y:(double)y + phase:(NSString *)phase + edge:(NSInteger)edge + error:(NSError * _Nullable __autoreleasing *)error { + return [self.session sendEdgeTouchWithNormalizedX:x + normalizedY:y + phase:phase + edge:edge + error:error]; +} + - (BOOL)sendMultiTouchAtX1:(double)x1 y1:(double)y1 x2:(double)x2 @@ -141,6 +153,24 @@ - (BOOL)pressHome:(NSError * _Nullable __autoreleasing *)error { return [self.session pressHomeButton:error]; } +- (BOOL)pressHardwareButtonNamed:(NSString *)buttonName + durationMs:(NSUInteger)durationMs + error:(NSError * _Nullable __autoreleasing *)error { + return [self.session pressHardwareButtonNamed:buttonName durationMs:durationMs error:error]; +} + +- (BOOL)sendHardwareButtonNamed:(NSString *)buttonName + pressed:(BOOL)pressed + usagePage:(NSNumber *)usagePage + usage:(NSNumber *)usage + error:(NSError * _Nullable __autoreleasing *)error { + return [self.session sendHardwareButtonNamed:buttonName + pressed:pressed + usagePage:usagePage + usage:usage + error:error]; +} + - (BOOL)openAppSwitcher:(NSError * _Nullable __autoreleasing *)error { return [self.session openAppSwitcher:error]; } diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts index 8cc699ab..1abc0c1f 100644 --- a/client/src/api/controls.ts +++ b/client/src/api/controls.ts @@ -1,8 +1,11 @@ import { accessTokenFromLocation, apiRequest } from "./client"; import { apiUrl } from "./config"; import type { + ButtonPayload, + EdgeTouchPayload, KeyPayload, LaunchPayload, + MultiTouchPayload, OpenUrlPayload, SimulatorMetadata, SimulatorResponse, @@ -11,7 +14,10 @@ import type { export type ControlMessage = | ({ type: "touch" } & TouchPayload) + | ({ type: "edgeTouch" } & EdgeTouchPayload) + | ({ type: "multiTouch" } & MultiTouchPayload) | ({ type: "key" } & KeyPayload) + | ({ type: "button" } & ButtonPayload) | { type: "dismissKeyboard" } | { type: "home" } | { type: "appSwitcher" } @@ -22,7 +28,12 @@ export type ControlMessage = async function postSimulatorAction( udid: string, action: string, - payload?: KeyPayload | LaunchPayload | OpenUrlPayload | TouchPayload, + payload?: + | ButtonPayload + | KeyPayload + | LaunchPayload + | OpenUrlPayload + | TouchPayload, ): Promise { const response = await apiRequest( `/api/simulators/${udid}/${action}`, @@ -84,6 +95,10 @@ export function pressHome(udid: string) { return postSimulatorAction(udid, "home"); } +export function pressSimulatorButton(udid: string, payload: ButtonPayload) { + return postSimulatorAction(udid, "button", payload); +} + export function openAppSwitcher(udid: string) { return postSimulatorAction(udid, "app-switcher"); } diff --git a/client/src/api/types.ts b/client/src/api/types.ts index b775516f..e45e625f 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -62,6 +62,27 @@ export interface ChromeProfile { screenHeight: number; cornerRadius: number; hasScreenMask?: boolean; + buttons?: ChromeButtonProfile[]; +} + +export interface ChromeButtonProfile { + name: string; + label?: string; + type?: string; + imageName?: string; + imageDownName?: string; + imageDownDrawMode?: string; + x: number; + y: number; + width: number; + height: number; + anchor?: "left" | "right" | "top" | "bottom" | string; + align?: string; + onTop?: boolean; + usagePage?: number; + usage?: number; + normalOffset?: { x: number; y: number }; + rolloverOffset?: { x: number; y: number }; } export interface AccessibilityFrame { @@ -187,11 +208,31 @@ export interface TouchPayload { phase: TouchPhase; } +export interface EdgeTouchPayload extends TouchPayload { + edge: "left" | "top" | "bottom" | "right" | "none"; +} + +export interface MultiTouchPayload { + x1: number; + y1: number; + x2: number; + y2: number; + phase: TouchPhase; +} + export interface KeyPayload { keyCode: number; modifiers: number; } +export interface ButtonPayload { + button: string; + durationMs?: number; + phase?: "down" | "up" | "began" | "ended" | "cancelled"; + usagePage?: number; + usage?: number; +} + export interface LaunchPayload { bundleId: string; } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index c4c4666b..2a7aab20 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -22,6 +22,7 @@ import { openAppSwitcher, openSimulatorUrl, pressHome, + pressSimulatorButton, rotateLeft, rotateRight, simulatorControlSocketUrl, @@ -142,10 +143,28 @@ interface StreamQualityResponse { videoCodec?: string; } -function buildChromeUrl(udid: string, stamp: number): string { +function buildChromeUrl( + udid: string, + stamp: number, + includeButtons = true, +): string { return buildAuthenticatedAssetUrl( `/api/simulators/${udid}/chrome.png`, stamp, + includeButtons ? undefined : { buttons: "false" }, + ); +} + +function buildChromeButtonUrl( + udid: string, + button: string, + pressed: boolean, + stamp: number, +): string { + return buildAuthenticatedAssetUrl( + `/api/simulators/${udid}/chrome-button/${encodeURIComponent(button)}.png`, + stamp, + pressed ? { pressed: "true" } : undefined, ); } @@ -156,9 +175,16 @@ function buildScreenMaskUrl(udid: string, stamp: number): string { ); } -function buildAuthenticatedAssetUrl(path: string, stamp: number): string { +function buildAuthenticatedAssetUrl( + path: string, + stamp: number, + params?: Record, +): string { const url = new URL(apiUrl(path), window.location.href); url.searchParams.set("stamp", String(stamp)); + for (const [key, value] of Object.entries(params ?? {})) { + url.searchParams.set(key, value); + } const token = accessTokenFromLocation(); if (token) { url.searchParams.set("simdeckToken", token); @@ -688,9 +714,28 @@ export function AppShell({ const autoViewportOffsetY = viewMode === "manual" ? 0 : -zoomDockReservedHeight / 2; const screenAspect = screenAspectRatio(effectiveDeviceNaturalSize); + const chromeHasInteractiveButtons = Boolean( + viewportChromeProfile?.buttons?.length, + ); const chromeUrl = selectedSimulator - ? buildChromeUrl(selectedSimulator.udid, streamStamp) + ? buildChromeUrl( + selectedSimulator.udid, + streamStamp, + !chromeHasInteractiveButtons, + ) : ""; + const chromeButtonUrl = useCallback( + (button: string, pressed = false) => + selectedSimulator + ? buildChromeButtonUrl( + selectedSimulator.udid, + button, + pressed, + streamStamp, + ) + : "", + [selectedSimulator?.udid, streamStamp], + ); const chromeRequired = Boolean( (shouldRenderChrome && !chromeProfileReady) || (viewportChromeProfile && chromeUrl), @@ -1126,7 +1171,40 @@ export function AppShell({ } sendTouchControl(selectedSimulator.udid, phase, coords); }, + onEdgeTouch: (phase, coords, edge) => { + if (!selectedSimulator) { + return; + } + if (phase === "began" && accessibilitySelectedId) { + setAccessibilitySelectedId(""); + setAccessibilityHoveredId(null); + } + sendControl(selectedSimulator.udid, { + type: "edgeTouch", + ...coords, + phase, + edge, + }); + }, + onMultiTouch: (phase: TouchPhase, first: Point, second: Point) => { + if (!selectedSimulator) { + return; + } + if (phase === "began" && accessibilitySelectedId) { + setAccessibilitySelectedId(""); + setAccessibilityHoveredId(null); + } + sendControl(selectedSimulator.udid, { + type: "multiTouch", + x1: first.x, + y1: first.y, + x2: second.x, + y2: second.y, + phase, + }); + }, onTouchPreview: showTouchIndicator, + onMultiTouchPreview: showTouchIndicators, pan, rotationQuarterTurns, setPan, @@ -1487,11 +1565,22 @@ export function AppShell({ } function showTouchIndicator(phase: TouchPhase, coords: Point) { + showTouchIndicators(phase, [coords]); + } + + function showTouchIndicators(phase: TouchPhase, coords: Point[]) { if (!touchOverlayVisible) { return; } - setTouchIndicators([{ id: 1, phase, x: coords.x, y: coords.y }]); + setTouchIndicators( + coords.map((coord, index) => ({ + id: index + 1, + phase, + x: coord.x, + y: coord.y, + })), + ); if (touchIndicatorTimeoutRef.current) { clearTimeout(touchIndicatorTimeoutRef.current); touchIndicatorTimeoutRef.current = 0; @@ -1588,6 +1677,41 @@ export function AppShell({ ); } + function sendHardwareButtonEvent( + button: string, + phase: "down" | "up", + usagePage?: number, + usage?: number, + ) { + if (!selectedSimulator) { + return; + } + if (phase === "down") { + setAccessibilitySelectedId(""); + setAccessibilityHoveredId(null); + } + if ( + !sendControl(selectedSimulator.udid, { + type: "button", + button, + phase, + usagePage, + usage, + }) + ) { + void runAction( + () => + pressSimulatorButton(selectedSimulator.udid, { + button, + phase, + usagePage, + usage, + }), + false, + ); + } + } + async function submitPairing(event: FormEvent) { event.preventDefault(); const code = pairingCode.trim(); @@ -1822,6 +1946,7 @@ export function AppShell({ chromeRequired={chromeRequired} chromeScreenStyle={viewportScreenStyle} chromeUrl={chromeUrl} + chromeButtonUrl={chromeButtonUrl} debugPanel={ debugVisible ? ( setChromeLoaded(true)} + onChromeButtonEvent={sendHardwareButtonEvent} onPanPointerMove={pointerInput.handlePanPointerMove} onPanPointerUp={pointerInput.handlePanPointerUp} onPickerHover={setAccessibilityHoveredId} diff --git a/client/src/features/input/usePointerInput.ts b/client/src/features/input/usePointerInput.ts index 8e45e57a..8c51f040 100644 --- a/client/src/features/input/usePointerInput.ts +++ b/client/src/features/input/usePointerInput.ts @@ -1,7 +1,10 @@ import { useRef, useState } from "react"; import type { ChromeProfile, TouchPhase } from "../../api/types"; -import { normalizedPointerCoordinatesForOrientation } from "./gestureMath"; +import { + normalizedPointerCoordinates, + normalizedPointerCoordinatesForOrientation, +} from "./gestureMath"; import { clampPan } from "../viewport/viewportMath"; import type { Point, Size } from "../viewport/types"; @@ -16,9 +19,31 @@ interface UsePointerInputOptions { rotationQuarterTurns: number; setPan: React.Dispatch>; onTouch: (phase: TouchPhase, coords: Point) => void; + onEdgeTouch?: ( + phase: TouchPhase, + coords: Point, + edge: "left" | "top" | "bottom" | "right" | "none", + ) => void; + onMultiTouch?: (phase: TouchPhase, first: Point, second: Point) => void; onTouchPreview?: (phase: TouchPhase, coords: Point) => void; + onMultiTouchPreview?: (phase: TouchPhase, coords: Point[]) => void; } +type ActiveGesture = + | { kind: "single"; pointerId: number } + | { kind: "edgeBottom"; pointerId: number } + | { kind: "pinch"; pointerId: number; first: Point; second: Point } + | { + kind: "twoFingerPan"; + pointerId: number; + start: Point; + first: Point; + second: Point; + }; + +const TWO_FINGER_SPREAD = 0.16; +const BOTTOM_EDGE_GESTURE_START_Y = 0.93; + export function usePointerInput({ canvasSize, chromeProfile, @@ -30,9 +55,12 @@ export function usePointerInput({ rotationQuarterTurns, setPan, onTouch, + onEdgeTouch, + onMultiTouch, onTouchPreview, + onMultiTouchPreview, }: UsePointerInputOptions) { - const activePointerRef = useRef(null); + const activeGestureRef = useRef(null); const panningRef = useRef<{ startX: number; startY: number; @@ -41,6 +69,30 @@ export function usePointerInput({ } | null>(null); const [isPanning, setIsPanning] = useState(false); + function clampPoint(point: Point): Point { + return { + x: Math.min(Math.max(point.x, 0), 1), + y: Math.min(Math.max(point.y, 0), 1), + }; + } + + function mirrorAroundCenter(point: Point): Point { + return clampPoint({ x: 1 - point.x, y: 1 - point.y }); + } + + function previewMultiTouch(phase: TouchPhase, first: Point, second: Point) { + if (onMultiTouchPreview) { + onMultiTouchPreview(phase, [first, second]); + return; + } + onTouchPreview?.(phase, first); + } + + function sendMultiTouch(phase: TouchPhase, first: Point, second: Point) { + previewMultiTouch(phase, first, second); + onMultiTouch?.(phase, first, second); + } + function startPanning(event: React.PointerEvent) { if (event.pointerType !== "mouse") { return; @@ -97,6 +149,7 @@ export function usePointerInput({ return; } event.stopPropagation(); + event.preventDefault(); const coords = normalizedPointerCoordinatesForOrientation( event, rotationQuarterTurns, @@ -104,24 +157,106 @@ export function usePointerInput({ if (!coords) { return; } - activePointerRef.current = event.pointerId; event.currentTarget.setPointerCapture(event.pointerId); + + const displayedCoords = normalizedPointerCoordinates(event); + if ( + displayedCoords && + displayedCoords.y >= BOTTOM_EDGE_GESTURE_START_Y && + !event.altKey && + onEdgeTouch + ) { + activeGestureRef.current = { + kind: "edgeBottom", + pointerId: event.pointerId, + }; + onTouchPreview?.("began", coords); + onEdgeTouch("began", coords, "bottom"); + return; + } + + if (event.altKey && onMultiTouch) { + if (event.shiftKey) { + const first = { x: 0.5 + TWO_FINGER_SPREAD / 2, y: 0.5 }; + const second = { x: 0.5 - TWO_FINGER_SPREAD / 2, y: 0.5 }; + activeGestureRef.current = { + kind: "twoFingerPan", + pointerId: event.pointerId, + start: coords, + first, + second, + }; + sendMultiTouch("began", first, second); + return; + } + + const first = clampPoint(coords); + const second = mirrorAroundCenter(first); + activeGestureRef.current = { + kind: "pinch", + pointerId: event.pointerId, + first, + second, + }; + sendMultiTouch("began", first, second); + return; + } + + activeGestureRef.current = { kind: "single", pointerId: event.pointerId }; onTouchPreview?.("began", coords); onTouch("began", coords); } function handleScreenPointerMove(event: React.PointerEvent) { event.stopPropagation(); - if (activePointerRef.current !== event.pointerId) { + const active = activeGestureRef.current; + if (!active || active.pointerId !== event.pointerId) { return; } + event.preventDefault(); const coords = normalizedPointerCoordinatesForOrientation( event, rotationQuarterTurns, ); - if (coords) { + if (!coords) { + return; + } + + if (active.kind === "pinch") { + const first = clampPoint(coords); + const second = mirrorAroundCenter(first); + activeGestureRef.current = { ...active, first, second }; + sendMultiTouch("moved", first, second); + return; + } + + if (active.kind === "twoFingerPan") { + const delta = { + x: coords.x - active.start.x, + y: coords.y - active.start.y, + }; + const first = clampPoint({ + x: 0.5 + TWO_FINGER_SPREAD / 2 + delta.x, + y: 0.5 + delta.y, + }); + const second = clampPoint({ + x: 0.5 - TWO_FINGER_SPREAD / 2 + delta.x, + y: 0.5 + delta.y, + }); + activeGestureRef.current = { ...active, first, second }; + sendMultiTouch("moved", first, second); + return; + } + + if (active.kind === "single") { onTouchPreview?.("moved", coords); onTouch("moved", coords); + return; + } + + if (active.kind === "edgeBottom") { + onTouchPreview?.("moved", coords); + onEdgeTouch?.("moved", coords, "bottom"); } } @@ -130,14 +265,37 @@ export function usePointerInput({ phase: Exclude, ) { event.stopPropagation(); - if (activePointerRef.current !== event.pointerId) { + const active = activeGestureRef.current; + if (!active || active.pointerId !== event.pointerId) { return; } - activePointerRef.current = null; + event.preventDefault(); + activeGestureRef.current = null; const coords = normalizedPointerCoordinatesForOrientation( event, rotationQuarterTurns, ); + + if (active.kind === "pinch") { + const first = coords ? clampPoint(coords) : active.first; + const second = coords ? mirrorAroundCenter(first) : active.second; + sendMultiTouch(phase, first, second); + return; + } + + if (active.kind === "twoFingerPan") { + sendMultiTouch(phase, active.first, active.second); + return; + } + + if (active.kind === "edgeBottom") { + if (coords) { + onTouchPreview?.(phase, coords); + onEdgeTouch?.(phase, coords, "bottom"); + } + return; + } + if (coords) { onTouchPreview?.(phase, coords); onTouch(phase, coords); diff --git a/client/src/features/viewport/DeviceChrome.tsx b/client/src/features/viewport/DeviceChrome.tsx index be333aa7..9ac199de 100644 --- a/client/src/features/viewport/DeviceChrome.tsx +++ b/client/src/features/viewport/DeviceChrome.tsx @@ -1,6 +1,10 @@ -import type { CSSProperties, Ref } from "react"; +import { useRef, useState, type CSSProperties, type Ref } from "react"; -import type { AccessibilityNode } from "../../api/types"; +import type { + AccessibilityNode, + ChromeButtonProfile, + ChromeProfile, +} from "../../api/types"; import { AccessibilityOverlay } from "../accessibility/AccessibilityOverlay"; import { findAccessibilityItemAtPoint } from "../accessibility/accessibilityTree"; import { normalizedPointerCoordinatesForOrientation } from "../input/gestureMath"; @@ -11,12 +15,20 @@ interface DeviceChromeProps { accessibilityPickerActive: boolean; accessibilityRoots: AccessibilityNode[]; accessibilitySelectedId: string; + chromeProfile: ChromeProfile | null; chromeScreenStyle: CSSProperties | null; chromeUrl: string; + chromeButtonUrl: (button: string, pressed?: boolean) => string; hasFrame: boolean; isBooted: boolean; isLoadingStream: boolean; isStreamError: boolean; + onChromeButtonEvent: ( + button: string, + phase: "down" | "up", + usagePage?: number, + usage?: number, + ) => void; onChromeLoad: () => void; onPanPointerCancel: (event: React.PointerEvent) => void; onPanPointerMove: (event: React.PointerEvent) => void; @@ -46,12 +58,15 @@ export function DeviceChrome({ accessibilityPickerActive, accessibilityRoots, accessibilitySelectedId, + chromeProfile, chromeScreenStyle, chromeUrl, + chromeButtonUrl, hasFrame, isBooted, isLoadingStream, isStreamError, + onChromeButtonEvent, onChromeLoad, onPanPointerCancel, onPanPointerMove, @@ -85,6 +100,12 @@ export function DeviceChrome({ onPointerUp={onPanPointerUp} style={shellStyle ?? undefined} > + + = { + action: "action", + home: "home", + lock: "power", + mute: "mute", + power: "power", + "side-button": "power", + "volume-down": "volume-down", + "volume-up": "volume-up", +}; + +function ChromeButtonOverlay({ + chromeButtonUrl, + chromeProfile, + layer, + onEvent, +}: { + chromeButtonUrl: (button: string, pressed?: boolean) => string; + chromeProfile: ChromeProfile | null; + layer: "under" | "over"; + onEvent: ( + button: string, + phase: "down" | "up", + usagePage?: number, + usage?: number, + ) => void; +}) { + const buttons = chromeProfile?.buttons ?? []; + if (!chromeProfile || buttons.length === 0) { + return null; + } + + return ( +
+ {buttons.map((button) => { + const onTop = Boolean(button.onTop); + if ((layer === "over") !== onTop) { + return null; + } + const wireName = wireButtonName(button); + if (!wireName) { + return null; + } + return ( + + ); + })} +
+ ); +} + +function ChromeButtonHitTarget({ + button, + chromeButtonUrl, + onEvent, + totalHeight, + totalWidth, + wireName, +}: { + button: ChromeButtonProfile; + chromeButtonUrl: (button: string, pressed?: boolean) => string; + onEvent: ( + button: string, + phase: "down" | "up", + usagePage?: number, + usage?: number, + ) => void; + totalHeight: number; + totalWidth: number; + wireName: string; +}) { + const pressedRef = useRef(false); + const [pressed, setPressed] = useState(false); + const label = button.label || humanizeChromeButtonName(button.name); + const rolloverDelta = button.rolloverOffset + ? { + x: button.rolloverOffset.x - (button.normalOffset?.x ?? 0), + y: button.rolloverOffset.y - (button.normalOffset?.y ?? 0), + } + : { x: 0, y: 0 }; + const imageUrl = chromeButtonUrl(button.name, false); + const pressedImageUrl = button.imageDownName + ? chromeButtonUrl(button.name, true) + : ""; + const downCompositeUnder = + pressed && + pressedImageUrl && + button.imageDownDrawMode?.toLowerCase() === "compositeunder"; + const style = { + height: `${(button.height / totalHeight) * 100}%`, + left: `${(button.x / totalWidth) * 100}%`, + top: `${(button.y / totalHeight) * 100}%`, + width: `${(button.width / totalWidth) * 100}%`, + "--button-rest-x": `${(rolloverDelta.x / Math.max(button.width, 1)) * 100}%`, + "--button-rest-y": `${(rolloverDelta.y / Math.max(button.height, 1)) * 100}%`, + "--button-hover-x": `${((rolloverDelta.x * 2) / Math.max(button.width, 1)) * 100}%`, + "--button-hover-y": `${((rolloverDelta.y * 2) / Math.max(button.height, 1)) * 100}%`, + } as CSSProperties & + Record< + | "--button-rest-x" + | "--button-rest-y" + | "--button-hover-x" + | "--button-hover-y", + string + >; + + function sendPhase(phase: "down" | "up") { + onEvent(wireName, phase, button.usagePage, button.usage); + } + + function endPress() { + if (!pressedRef.current) { + return; + } + pressedRef.current = false; + setPressed(false); + sendPhase("up"); + } + + return ( + + ); +} + +function wireButtonName(button: ChromeButtonProfile): string | null { + return CHROME_BUTTON_WIRE_NAMES[button.name.toLowerCase()] ?? null; +} + +function humanizeChromeButtonName(name: string) { + return name + .split(/[-_]/) + .map((part) => `${part.charAt(0).toUpperCase()}${part.slice(1)}`) + .join(" "); +} + interface ScreenLayerProps { accessibilityHoveredId: string | null; accessibilityPickerActive: boolean; diff --git a/client/src/features/viewport/SimulatorViewport.tsx b/client/src/features/viewport/SimulatorViewport.tsx index 19f74eaf..0385848e 100644 --- a/client/src/features/viewport/SimulatorViewport.tsx +++ b/client/src/features/viewport/SimulatorViewport.tsx @@ -30,6 +30,13 @@ interface SimulatorViewportProps { isLoading: boolean; isStreamError: boolean; isPanning: boolean; + onChromeButtonEvent: ( + button: string, + phase: "down" | "up", + usagePage?: number, + usage?: number, + ) => void; + chromeButtonUrl: (button: string, pressed?: boolean) => string; onChromeLoad: () => void; onPanPointerMove: (event: React.PointerEvent) => void; onPanPointerUp: () => void; @@ -83,6 +90,8 @@ export function SimulatorViewport({ isLoading, isStreamError, isPanning, + onChromeButtonEvent, + chromeButtonUrl, onChromeLoad, onPanPointerMove, onPanPointerUp, @@ -164,12 +173,15 @@ export function SimulatorViewport({ accessibilityPickerActive={accessibilityPickerActive} accessibilityRoots={accessibilityRoots} accessibilitySelectedId={accessibilitySelectedId} + chromeProfile={chromeProfile} chromeScreenStyle={chromeScreenStyle} chromeUrl={chromeUrl} + chromeButtonUrl={chromeButtonUrl} hasFrame={hasFrame} isBooted={selectedSimulator.isBooted} isLoadingStream={showScreenLoading} isStreamError={isStreamError} + onChromeButtonEvent={onChromeButtonEvent} onChromeLoad={onChromeLoad} onPanPointerCancel={onPanPointerUp} onPanPointerMove={onPanPointerMove} diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 184b3a80..b1b9fd8b 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -1227,6 +1227,64 @@ z-index: 2; } +.device-chrome-buttons { + position: absolute; + inset: 0; + z-index: 1; + pointer-events: none; +} + +.device-chrome-buttons-over { + z-index: 3; +} + +.device-chrome-button { + position: absolute; + display: block; + padding: 0; + border: 0; + background: transparent; + cursor: pointer; + pointer-events: auto; + touch-action: none; + transform: translate3d(var(--button-rest-x, 0), var(--button-rest-y, 0), 0); + transition: transform 130ms cubic-bezier(0.2, 0.8, 0.2, 1); + -webkit-tap-highlight-color: transparent; + z-index: 1; +} + +.device-chrome-button img { + position: absolute; + inset: 0; + display: block; + width: 100%; + height: 100%; + pointer-events: none; + user-select: none; + -webkit-user-drag: none; +} + +.device-chrome-button-image-under { + z-index: 0; +} + +.device-chrome-button-image { + z-index: 1; +} + +.device-chrome-button .device-chrome-button-preload { + display: none; +} + +.device-chrome-button:hover { + transform: translate3d(var(--button-hover-x, 0), var(--button-hover-y, 0), 0); +} + +.device-chrome-button:active, +.device-chrome-button.is-pressed { + transform: translate3d(var(--button-rest-x, 0), var(--button-rest-y, 0), 0); +} + .pan-enabled .device-bezel, .pan-enabled .device-shell { cursor: grab; diff --git a/docs/api/rest.md b/docs/api/rest.md index a0070e24..23eae55f 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -346,8 +346,21 @@ Content-Type: application/json { "button": "lock", "durationMs": 50 } ``` -Supported button names match the CLI: `home`, `lock`, `side-button`, `siri`, -and `apple-pay`. `durationMs` defaults to `0`. +Supported button names match the CLI and chrome controls: `home`, `lock`, +`power`, `side-button`, `volume-up`, `volume-down`, `action`, `mute`, +`app-switcher`, `siri`, and `apple-pay`. `durationMs` defaults to `0` and is +used for press-and-hold interactions. + +For live chrome interactions, send explicit button edges instead of a completed +press: + +```json +{ "button": "power", "phase": "down" } +``` + +`phase` accepts `down`, `up`, `began`, `ended`, and `cancelled`. Chrome controls +may also pass `usagePage` and `usage` from the device profile when an exact HID +usage is available. ### `POST /api/simulators/{udid}/home` @@ -383,15 +396,40 @@ Returns the bezel layout for the simulator: "screenY": 35, "screenWidth": 1170, "screenHeight": 2532, - "cornerRadius": 220 + "cornerRadius": 220, + "buttons": [ + { + "name": "power", + "label": "Sleep/Wake", + "x": 1210, + "y": 420, + "width": 18, + "height": 112, + "anchor": "right", + "onTop": false, + "normalOffset": { "x": -2, "y": 420 }, + "rolloverOffset": { "x": -4, "y": 420 }, + "imageName": "SideButton", + "imageDownName": "SideButtonPressed" + } + ] } ``` -The browser client uses this to compose chrome around the live frame. +The browser client uses this to compose chrome around the live frame and to +render physical button sprites over or under the device body. ### `GET /api/simulators/{udid}/chrome.png` -Returns the rendered bezel as a PNG. Cache headers are set to `no-cache, no-store, must-revalidate` so changes (e.g. after a device rotation) are picked up immediately. +Returns the rendered bezel as a PNG. Pass `?buttons=false` to omit physical +button sprites when the client renders them interactively. Cache headers are set +to `no-cache, no-store, must-revalidate` so changes (e.g. after a device +rotation) are picked up immediately. + +### `GET /api/simulators/{udid}/chrome-button/{button}.png` + +Returns a rendered physical button sprite. Pass `?pressed=true` for the +pressed-state sprite when the device profile exposes one. ## Accessibility diff --git a/docs/cli/commands.md b/docs/cli/commands.md index f67a127c..7ad378be 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -228,6 +228,8 @@ simdeck key-combo --modifiers cmd --key a simdeck type "hello" simdeck type --file message.txt simdeck button lock --duration-ms 1000 +simdeck button volume-up +simdeck button action --duration-ms 1000 simdeck dismiss-keyboard simdeck home simdeck app-switcher diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 36f05fd1..2769b739 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -269,10 +269,30 @@ pub(crate) enum ControlMessage { y: f64, phase: String, }, + EdgeTouch { + x: f64, + y: f64, + phase: String, + edge: String, + }, + MultiTouch { + x1: f64, + y1: f64, + x2: f64, + y2: f64, + phase: String, + }, Key { key_code: u16, modifiers: Option, }, + Button { + button: String, + duration_ms: Option, + phase: Option, + usage_page: Option, + usage: Option, + }, DismissKeyboard, Home, AppSwitcher, @@ -294,6 +314,20 @@ struct ButtonPayload { button: String, #[serde(rename = "durationMs")] duration_ms: Option, + phase: Option, + #[serde(rename = "usagePage")] + usage_page: Option, + usage: Option, +} + +#[derive(Deserialize)] +struct ChromePngQuery { + buttons: Option, +} + +#[derive(Deserialize)] +struct ChromeButtonPngQuery { + pressed: Option, } #[derive(Deserialize, Clone, Default)] @@ -552,6 +586,10 @@ pub fn router(state: AppState) -> Router { .route("/api/simulators/{udid}/rotate-right", post(rotate_right)) .route("/api/simulators/{udid}/chrome-profile", get(chrome_profile)) .route("/api/simulators/{udid}/chrome.png", get(chrome_png)) + .route( + "/api/simulators/{udid}/chrome-button/{button}", + get(chrome_button_png), + ) .route( "/api/simulators/{udid}/screen-mask.png", get(screen_mask_png), @@ -1656,7 +1694,25 @@ async fn run_control_queue( } fn control_message_is_move(message: &ControlMessage) -> bool { - matches!(message, ControlMessage::Touch { phase, .. } if phase == "moved") + matches!( + message, + ControlMessage::Touch { phase, .. } + | ControlMessage::EdgeTouch { phase, .. } + | ControlMessage::MultiTouch { phase, .. } + if phase == "moved" + ) +} + +fn edge_name_to_hid_value(edge: &str) -> Option { + let edge = edge.trim().to_ascii_lowercase(); + match edge.as_str() { + "left" => Some(1), + "top" => Some(2), + "bottom" => Some(3), + "right" => Some(4), + "none" => Some(0), + _ => None, + } } pub(crate) async fn run_control_message( @@ -1672,10 +1728,63 @@ pub(crate) async fn run_control_message( } session.send_touch(x.clamp(0.0, 1.0), y.clamp(0.0, 1.0), &phase) } + ControlMessage::EdgeTouch { x, y, phase, edge } => { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + let edge = edge_name_to_hid_value(edge.as_str()).ok_or_else(|| { + AppError::bad_request("`edge` must be `left`, `top`, `bottom`, `right`, or `none`.") + })?; + session.send_edge_touch(x.clamp(0.0, 1.0), y.clamp(0.0, 1.0), &phase, edge) + } + ControlMessage::MultiTouch { + x1, + y1, + x2, + y2, + phase, + } => { + if !x1.is_finite() || !y1.is_finite() || !x2.is_finite() || !y2.is_finite() { + return Err(AppError::bad_request( + "`x1`, `y1`, `x2`, and `y2` must be finite normalized numbers.", + )); + } + session.send_multitouch( + x1.clamp(0.0, 1.0), + y1.clamp(0.0, 1.0), + x2.clamp(0.0, 1.0), + y2.clamp(0.0, 1.0), + &phase, + ) + } ControlMessage::Key { key_code, modifiers, } => session.send_key(key_code, modifiers.unwrap_or(0)), + ControlMessage::Button { + button, + duration_ms, + phase, + usage_page, + usage, + } => { + if let Some(phase) = phase { + let pressed = match phase.as_str() { + "down" | "began" => true, + "up" | "ended" | "cancelled" => false, + _ => { + return Err(AppError::bad_request( + "`phase` must be `down`, `up`, `began`, `ended`, or `cancelled`.", + )) + } + }; + session.send_button(&button, pressed, usage_page, usage) + } else { + session.press_button(&button, duration_ms.unwrap_or(0)) + } + } ControlMessage::DismissKeyboard => session.send_key(41, 0), ControlMessage::Home => session.press_home(), ControlMessage::AppSwitcher => session.open_app_switcher(), @@ -1748,10 +1857,32 @@ async fn press_button( if payload.button.trim().is_empty() { return Err(AppError::bad_request("Request body must include `button`.")); } - run_bridge_action(state, move |bridge| { - bridge.press_button(&udid, &payload.button, payload.duration_ms.unwrap_or(0)) - }) - .await?; + if let Some(phase) = payload.phase.as_deref() { + let pressed = match phase { + "down" | "began" => true, + "up" | "ended" | "cancelled" => false, + _ => { + return Err(AppError::bad_request( + "`phase` must be `down`, `up`, `began`, `ended`, or `cancelled`.", + )) + } + }; + run_bridge_action(state, move |bridge| { + bridge.send_button( + &udid, + &payload.button, + pressed, + payload.usage_page, + payload.usage, + ) + }) + .await?; + } else { + run_bridge_action(state, move |bridge| { + bridge.press_button(&udid, &payload.button, payload.duration_ms.unwrap_or(0)) + }) + .await?; + } Ok(json(json_value!({ "ok": true }))) } @@ -1798,8 +1929,66 @@ async fn chrome_profile( async fn chrome_png( State(state): State, Path(udid): Path, + Query(query): Query, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { - let png = run_bridge_action(state, move |bridge| bridge.chrome_png(&udid)).await?; + let include_buttons = query + .buttons + .as_deref() + .map(|value| !value.eq_ignore_ascii_case("false")) + .unwrap_or(true); + let png = run_bridge_action(state, move |bridge| { + bridge.chrome_png_with_buttons(&udid, include_buttons) + }) + .await?; + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); + headers.insert( + header::CACHE_CONTROL, + "no-cache, no-store, must-revalidate".parse().unwrap(), + ); + Ok((StatusCode::OK, headers, png)) +} + +fn parse_asset_bool(value: &str) -> bool { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) +} + +async fn chrome_button_png( + State(state): State, + Path((udid, button)): Path<(String, String)>, + Query(query): Query, +) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + let button_name = button.strip_suffix(".png").unwrap_or(&button).to_owned(); + let png = if let Some(pressed) = query.pressed.as_deref().map(parse_asset_bool) { + run_bridge_action(state, move |bridge| { + bridge.chrome_button_png(&udid, &button_name, pressed) + }) + .await? + } else if let Some(base_name) = button_name.strip_suffix("-down").map(str::to_owned) { + let exact_udid = udid.clone(); + let exact_name = button_name.clone(); + match run_bridge_action(state.clone(), move |bridge| { + bridge.chrome_button_png(&exact_udid, &exact_name, false) + }) + .await + { + Ok(png) => png, + Err(_) => { + run_bridge_action(state, move |bridge| { + bridge.chrome_button_png(&udid, &base_name, true) + }) + .await? + } + } + } else { + run_bridge_action(state, move |bridge| { + bridge.chrome_button_png(&udid, &button_name, false) + }) + .await? + }; let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); headers.insert( diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs index 84946b88..0cf186f1 100644 --- a/server/src/native/bridge.rs +++ b/server/src/native/bridge.rs @@ -117,6 +117,47 @@ pub struct ChromeProfile { pub corner_radius: f64, #[serde(rename = "hasScreenMask", default)] pub has_screen_mask: bool, + #[serde(default)] + pub buttons: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChromeButtonProfile { + pub name: String, + #[serde(default)] + pub label: Option, + #[serde(rename = "type", default)] + pub button_type: Option, + #[serde(rename = "imageName", default)] + pub image_name: Option, + #[serde(rename = "imageDownName", default)] + pub image_down_name: Option, + #[serde(rename = "imageDownDrawMode", default)] + pub image_down_draw_mode: Option, + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, + #[serde(default)] + pub anchor: Option, + #[serde(default)] + pub align: Option, + #[serde(rename = "onTop", default)] + pub on_top: bool, + #[serde(rename = "usagePage", default)] + pub usage_page: Option, + #[serde(default)] + pub usage: Option, + #[serde(rename = "normalOffset", default)] + pub normal_offset: Option, + #[serde(rename = "rolloverOffset", default)] + pub rollover_offset: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChromeButtonOffset { + pub x: f64, + pub y: f64, } #[derive(Default, Clone)] @@ -201,11 +242,44 @@ impl NativeBridge { serde_json::from_str(&json).map_err(|e| AppError::internal(e.to_string())) } - pub fn chrome_png(&self, udid: &str) -> Result, AppError> { + pub fn chrome_png_with_buttons( + &self, + udid: &str, + include_buttons: bool, + ) -> Result, AppError> { + let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + let bytes = + ffi::xcw_native_render_chrome_png(udid.as_ptr(), include_buttons, &mut error); + if bytes.data.is_null() { + return Err( + take_error(error).unwrap_or_else(|| AppError::native("Unknown native error.")) + ); + } + let data = std::slice::from_raw_parts(bytes.data, bytes.length).to_vec(); + ffi::xcw_native_free_bytes(bytes); + Ok(data) + } + } + + pub fn chrome_button_png( + &self, + udid: &str, + button_name: &str, + pressed: bool, + ) -> Result, AppError> { let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; + let button_name = + CString::new(button_name).map_err(|e| AppError::bad_request(e.to_string()))?; unsafe { let mut error = ptr::null_mut(); - let bytes = ffi::xcw_native_render_chrome_png(udid.as_ptr(), &mut error); + let bytes = ffi::xcw_native_render_chrome_button_png( + udid.as_ptr(), + button_name.as_ptr(), + pressed, + &mut error, + ); if bytes.data.is_null() { return Err( take_error(error).unwrap_or_else(|| AppError::native("Unknown native error.")) @@ -371,6 +445,34 @@ impl NativeBridge { } } + pub fn send_button( + &self, + udid: &str, + button: &str, + pressed: bool, + usage_page: Option, + usage: Option, + ) -> Result<(), AppError> { + let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; + let button = CString::new(button).map_err(|e| AppError::bad_request(e.to_string()))?; + let has_usage = usage_page.is_some() && usage.is_some(); + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_send_button( + udid.as_ptr(), + button.as_ptr(), + pressed, + has_usage, + usage_page.unwrap_or(0), + usage.unwrap_or(0), + &mut error, + ), + error, + ) + } + } + pub fn rotate_right(&self, udid: &str) -> Result<(), AppError> { let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; unsafe { @@ -691,6 +793,50 @@ impl NativeSession { } } + pub fn send_edge_touch(&self, x: f64, y: f64, phase: &str, edge: u32) -> Result<(), AppError> { + let phase = CString::new(phase).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_session_send_edge_touch( + self.handle, + x, + y, + phase.as_ptr(), + edge, + &mut error, + ), + error, + ) + } + } + + pub fn send_multitouch( + &self, + x1: f64, + y1: f64, + x2: f64, + y2: f64, + phase: &str, + ) -> Result<(), AppError> { + let phase = CString::new(phase).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_session_send_multitouch( + self.handle, + x1, + y1, + x2, + y2, + phase.as_ptr(), + &mut error, + ), + error, + ) + } + } + pub fn send_key(&self, key_code: u16, modifiers: u32) -> Result<(), AppError> { unsafe { let mut error = ptr::null_mut(); @@ -711,6 +857,48 @@ impl NativeSession { } } + pub fn press_button(&self, button: &str, duration_ms: u32) -> Result<(), AppError> { + let button = CString::new(button).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_session_press_button( + self.handle, + button.as_ptr(), + duration_ms, + &mut error, + ), + error, + ) + } + } + + pub fn send_button( + &self, + button: &str, + pressed: bool, + usage_page: Option, + usage: Option, + ) -> Result<(), AppError> { + let button = CString::new(button).map_err(|e| AppError::bad_request(e.to_string()))?; + let has_usage = usage_page.is_some() && usage.is_some(); + unsafe { + let mut error = ptr::null_mut(); + bool_result( + ffi::xcw_native_session_send_button( + self.handle, + button.as_ptr(), + pressed, + has_usage, + usage_page.unwrap_or(0), + usage.unwrap_or(0), + &mut error, + ), + error, + ) + } + } + pub fn open_app_switcher(&self) -> Result<(), AppError> { unsafe { let mut error = ptr::null_mut(); diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index e87d8fb4..2d0659fb 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -60,6 +60,13 @@ unsafe extern "C" { ) -> *mut c_char; pub fn xcw_native_render_chrome_png( udid: *const c_char, + include_buttons: bool, + error_message: *mut *mut c_char, + ) -> xcw_native_owned_bytes; + pub fn xcw_native_render_chrome_button_png( + udid: *const c_char, + button_name: *const c_char, + pressed: bool, error_message: *mut *mut c_char, ) -> xcw_native_owned_bytes; pub fn xcw_native_render_screen_mask_png( @@ -108,6 +115,15 @@ unsafe extern "C" { duration_ms: u32, error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_send_button( + udid: *const c_char, + button_name: *const c_char, + pressed: bool, + has_usage: bool, + usage_page: u32, + usage: u32, + error_message: *mut *mut c_char, + ) -> bool; pub fn xcw_native_rotate_right(udid: *const c_char, error_message: *mut *mut c_char) -> bool; pub fn xcw_native_rotate_left(udid: *const c_char, error_message: *mut *mut c_char) -> bool; pub fn xcw_native_erase_simulator(udid: *const c_char, error_message: *mut *mut c_char) @@ -197,6 +213,23 @@ unsafe extern "C" { phase: *const c_char, error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_session_send_edge_touch( + handle: *mut c_void, + x: f64, + y: f64, + phase: *const c_char, + edge: u32, + error_message: *mut *mut c_char, + ) -> bool; + pub fn xcw_native_session_send_multitouch( + handle: *mut c_void, + x1: f64, + y1: f64, + x2: f64, + y2: f64, + phase: *const c_char, + error_message: *mut *mut c_char, + ) -> bool; pub fn xcw_native_session_send_key( handle: *mut c_void, key_code: u16, @@ -207,6 +240,21 @@ unsafe extern "C" { handle: *mut c_void, error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_session_press_button( + handle: *mut c_void, + button_name: *const c_char, + duration_ms: u32, + error_message: *mut *mut c_char, + ) -> bool; + pub fn xcw_native_session_send_button( + handle: *mut c_void, + button_name: *const c_char, + pressed: bool, + has_usage: bool, + usage_page: u32, + usage: u32, + error_message: *mut *mut c_char, + ) -> bool; pub fn xcw_native_session_open_app_switcher( handle: *mut c_void, error_message: *mut *mut c_char, diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs index a274d73c..dea84b4d 100644 --- a/server/src/simulators/session.rs +++ b/server/src/simulators/session.rs @@ -11,18 +11,18 @@ use std::sync::{Arc, Condvar, Mutex, RwLock, Weak}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::broadcast; use tokio::task; -use tokio::time::{sleep, timeout, Instant}; +use tokio::time::{sleep_until, timeout, Instant}; use tracing::debug; // This channel carries encoded H.264 access units. Subscribers must not miss // ordinary P-frames: dropping compressed references creates decoder artifacts // even on a perfect localhost link. Coalescing is only safe before encoding. const FRAME_BROADCAST_CAPACITY: usize = 128; -const MIN_REFRESH_INTERVAL_MS: u64 = 16; const MIN_KEYFRAME_INTERVAL_MS: u64 = 250; const DEFAULT_SHARED_REFRESH_FPS: u64 = 60; const MIN_SHARED_REFRESH_FPS: u64 = 15; const MAX_SHARED_REFRESH_FPS: u64 = 240; +const MIN_REFRESH_INTERVAL_US: u64 = 1_000_000 / MAX_SHARED_REFRESH_FPS; pub struct SimulatorSession { inner: Arc, @@ -41,7 +41,7 @@ struct SimulatorSessionInner { display_width: AtomicU64, display_height: AtomicU64, frame_sequence: AtomicU64, - last_refresh_ms: AtomicU64, + last_refresh_us: AtomicU64, last_keyframe_ms: AtomicU64, active_frame_subscribers: AtomicU64, refresh_pump_running: AtomicBool, @@ -93,7 +93,7 @@ impl SimulatorSession { display_width: AtomicU64::new(0), display_height: AtomicU64::new(0), frame_sequence: AtomicU64::new(0), - last_refresh_ms: AtomicU64::new(0), + last_refresh_us: AtomicU64::new(0), last_keyframe_ms: AtomicU64::new(0), active_frame_subscribers: AtomicU64::new(0), refresh_pump_running: AtomicBool::new(false), @@ -206,7 +206,9 @@ impl SimulatorSession { return; } self.inner.last_keyframe_ms.store(now, Ordering::Relaxed); - self.inner.last_refresh_ms.store(now, Ordering::Relaxed); + self.inner + .last_refresh_us + .store(now_us(), Ordering::Relaxed); self.inner .metrics .keyframe_requests @@ -217,7 +219,9 @@ impl SimulatorSession { fn request_keyframe_immediate(&self) { let now = now_ms(); self.inner.last_keyframe_ms.store(now, Ordering::Relaxed); - self.inner.last_refresh_ms.store(now, Ordering::Relaxed); + self.inner + .last_refresh_us + .store(now_us(), Ordering::Relaxed); self.inner .metrics .keyframe_requests @@ -228,7 +232,7 @@ impl SimulatorSession { pub fn reconfigure_video_encoder(&self) { *self.inner.latest_keyframe.write().unwrap() = None; self.inner.last_keyframe_ms.store(0, Ordering::Relaxed); - self.inner.last_refresh_ms.store(0, Ordering::Relaxed); + self.inner.last_refresh_us.store(0, Ordering::Relaxed); self.inner.native.reconfigure_video_encoder(); } @@ -236,6 +240,21 @@ impl SimulatorSession { self.inner.native.send_touch(x, y, phase) } + pub fn send_edge_touch(&self, x: f64, y: f64, phase: &str, edge: u32) -> Result<(), AppError> { + self.inner.native.send_edge_touch(x, y, phase, edge) + } + + pub fn send_multitouch( + &self, + x1: f64, + y1: f64, + x2: f64, + y2: f64, + phase: &str, + ) -> Result<(), AppError> { + self.inner.native.send_multitouch(x1, y1, x2, y2, phase) + } + pub fn send_key(&self, key_code: u16, modifiers: u32) -> Result<(), AppError> { self.inner.native.send_key(key_code, modifiers) } @@ -244,6 +263,22 @@ impl SimulatorSession { self.inner.native.press_home() } + pub fn press_button(&self, button: &str, duration_ms: u32) -> Result<(), AppError> { + self.inner.native.press_button(button, duration_ms) + } + + pub fn send_button( + &self, + button: &str, + pressed: bool, + usage_page: Option, + usage: Option, + ) -> Result<(), AppError> { + self.inner + .native + .send_button(button, pressed, usage_page, usage) + } + pub fn open_app_switcher(&self) -> Result<(), AppError> { self.inner.native.open_app_switcher() } @@ -319,6 +354,7 @@ impl SimulatorSessionInner { let inner = self.clone(); tokio::spawn(async move { + let mut next_tick = Instant::now(); loop { if inner.active_frame_subscribers.load(Ordering::Relaxed) == 0 { inner.refresh_pump_running.store(false, Ordering::Release); @@ -335,18 +371,24 @@ impl SimulatorSessionInner { } inner.request_refresh(); - sleep(shared_refresh_interval()).await; + let refresh_interval = shared_refresh_interval(); + next_tick += refresh_interval; + let now = Instant::now(); + if next_tick <= now { + next_tick = now + refresh_interval; + } + sleep_until(next_tick).await; } }); } fn request_refresh(&self) { - let now = now_ms(); - let previous = self.last_refresh_ms.load(Ordering::Relaxed); - if now.saturating_sub(previous) < MIN_REFRESH_INTERVAL_MS { + let now = now_us(); + let previous = self.last_refresh_us.load(Ordering::Relaxed); + if now.saturating_sub(previous) < MIN_REFRESH_INTERVAL_US { return; } - self.last_refresh_ms.store(now, Ordering::Relaxed); + self.last_refresh_us.store(now, Ordering::Relaxed); self.native.request_refresh(); } @@ -435,12 +477,37 @@ fn now_ms() -> u64 { .as_millis() as u64 } +fn now_us() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_micros() as u64 +} + fn shared_refresh_interval() -> Duration { - let fps = std::env::var("SIMDECK_REALTIME_FPS") + let target_fps = std::env::var("SIMDECK_REALTIME_FPS") .or_else(|_| std::env::var("SIMDECK_LOCAL_STREAM_FPS")) .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(DEFAULT_SHARED_REFRESH_FPS) .clamp(MIN_SHARED_REFRESH_FPS, MAX_SHARED_REFRESH_FPS); + let fps = if realtime_stream_enabled() { + target_fps.saturating_mul(2) + } else { + target_fps + } + .clamp(MIN_SHARED_REFRESH_FPS, MAX_SHARED_REFRESH_FPS); Duration::from_micros(1_000_000 / fps) } + +fn realtime_stream_enabled() -> bool { + std::env::var("SIMDECK_REALTIME_STREAM") + .map(|value| { + let value = value.trim(); + value == "1" + || value.eq_ignore_ascii_case("true") + || value.eq_ignore_ascii_case("yes") + || value.eq_ignore_ascii_case("on") + }) + .unwrap_or(false) +} diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index c5c4eaaf..d0d13553 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -554,7 +554,13 @@ struct WebRtcStreamCommand { } fn webrtc_control_message_is_move(message: &ControlMessage) -> bool { - matches!(message, ControlMessage::Touch { phase, .. } if phase == "moved") + matches!( + message, + ControlMessage::Touch { phase, .. } + | ControlMessage::EdgeTouch { phase, .. } + | ControlMessage::MultiTouch { phase, .. } + if phase == "moved" + ) } fn is_h264_codec(codec: &str) -> bool { diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 4be83e55..dc02abcd 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -130,6 +130,10 @@ simdeck dismiss-keyboard simdeck button home simdeck button lock --duration-ms 1000 simdeck button side-button +simdeck button volume-up +simdeck button volume-down +simdeck button action --duration-ms 1000 +simdeck button mute simdeck button siri simdeck button apple-pay simdeck home