From 79f70ec39f5d943282431008d039a5d13252845d Mon Sep 17 00:00:00 2001 From: Yoni Samlan Date: Sat, 13 Jun 2026 12:44:48 -0400 Subject: [PATCH 1/2] fix: rotate synthesized iOS taps into native screen space --- .../RunnerSynthesizedGesture.h | 4 ++ .../RunnerSynthesizedGesture.m | 8 +++ .../RunnerTests+Interaction.swift | 64 +++++++++++++++++-- 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index a16c11223..bf0653dd7 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -25,6 +25,10 @@ NS_ASSUME_NONNULL_BEGIN x:(double)x y:(double)y; +// UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight, +// 4 landscapeLeft), or 0 if unreadable. ++ (NSInteger)interfaceOrientationForApplication:(id)application; + @end NS_ASSUME_NONNULL_END diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index 7f5aedf7e..4daaf9e81 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -131,6 +131,14 @@ + (NSString * _Nullable)synthesizeTapWithApplication:(id)application } } ++ (NSInteger)interfaceOrientationForApplication:(id)application { + SEL selector = NSSelectorFromString(@"interfaceOrientation"); + if (![application respondsToSelector:selector]) { + return 0; // UIInterfaceOrientationUnknown + } + return ((RunnerMsgSendInteger)objc_msgSend)(application, selector); +} + + (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application x:(double)x y:(double)y diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index eaf0ac8c3..9a50e025b 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -634,6 +634,31 @@ extension RunnerTests { return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration) } + /// Rotates an interface-oriented point into the device-native (portrait) space the + /// synthesized event path consumes — synthesized events skip XCTest's orientation + /// handling, so without this a landscape tap lands in the wrong place. + func nativeSynthesizedPoint( + orientedX x: Double, + orientedY y: Double, + in frame: CGRect, + interfaceOrientation: Int + ) -> CGPoint { + let localX = x - Double(frame.minX) + let localY = y - Double(frame.minY) + let width = Double(frame.width) + let height = Double(frame.height) + switch interfaceOrientation { + case 3: // landscapeRight + return CGPoint(x: height - localY, y: localX) + case 4: // landscapeLeft + return CGPoint(x: localY, y: width - localX) + case 2: // portraitUpsideDown + return CGPoint(x: width - localX, y: height - localY) + default: // 1 portrait, 0 unknown + return CGPoint(x: localX, y: localY) + } + } + func synthesizedDragAt( app: XCUIApplication, x: Double, @@ -643,12 +668,16 @@ extension RunnerTests { durationMs: Double ) -> RunnerInteractionOutcome { #if os(iOS) + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let frame = app.frame + let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation) + let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeSwipe( withApplication: app, - x: x, - y: y, - x2: x2, - y2: y2, + x: Double(start.x), + y: Double(start.y), + x2: Double(end.x), + y2: Double(end.y), durationMs: durationMs ) { return .unsupported( @@ -672,10 +701,12 @@ extension RunnerTests { func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { #if os(iOS) + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeTap( withApplication: app, - x: x, - y: y + x: Double(point.x), + y: Double(point.y) ) { return .unsupported( message: message, @@ -1091,4 +1122,25 @@ extension RunnerTests { let element = app.descendants(matching: .any).matching(predicate).firstMatch return element.exists ? element : nil } + + // Identity in portrait/unknown, 90° per landscape, 180° upside-down. + func testNativeSynthesizedPointRotatesByInterfaceOrientation() { + let portrait = CGRect(x: 0, y: 0, width: 834, height: 1210) + let landscape = CGRect(x: 0, y: 0, width: 1210, height: 834) + // (frame, UIInterfaceOrientation, expected native point) for a tap at (170, 268). + let cases: [(CGRect, Int, CGPoint)] = [ + (portrait, 1, CGPoint(x: 170, y: 268)), // portrait + (landscape, 3, CGPoint(x: 566, y: 170)), // landscapeRight + (landscape, 4, CGPoint(x: 268, y: 1040)), // landscapeLeft + (portrait, 2, CGPoint(x: 664, y: 942)), // portraitUpsideDown + (portrait, 0, CGPoint(x: 170, y: 268)), // unknown -> identity + ] + for (frame, orientation, expected) in cases { + XCTAssertEqual( + nativeSynthesizedPoint(orientedX: 170, orientedY: 268, in: frame, interfaceOrientation: orientation), + expected, + "interfaceOrientation \(orientation)" + ) + } + } } From a2a573dbe412e17232dfde5f7ec13450489fd450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 14 Jun 2026 10:03:43 +0200 Subject: [PATCH 2/2] fix: rotate synthesized iOS transform gestures --- examples/test-app/app.json | 2 +- examples/test-app/src/screens/GestureLab.tsx | 8 +- .../RunnerTests+Interaction.swift | 82 ++++++++++++++++--- 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/examples/test-app/app.json b/examples/test-app/app.json index 5404eb869..4b6f5a37b 100644 --- a/examples/test-app/app.json +++ b/examples/test-app/app.json @@ -3,7 +3,7 @@ "name": "Agent Device Tester", "slug": "agent-device-test-app", "version": "1.0.0", - "orientation": "portrait", + "orientation": "default", "userInterfaceStyle": "automatic", "newArchEnabled": true, "plugins": ["expo-router"], diff --git a/examples/test-app/src/screens/GestureLab.tsx b/examples/test-app/src/screens/GestureLab.tsx index 53a3863e3..64ba952a3 100644 --- a/examples/test-app/src/screens/GestureLab.tsx +++ b/examples/test-app/src/screens/GestureLab.tsx @@ -137,10 +137,13 @@ export function GestureLab() { const panChanged = Math.abs(transform.offsetX) > 0 || Math.abs(transform.offsetY) > 0; const pinchChanged = Math.abs(transform.scale - 1) > 0.01; const rotateChanged = rotationDegrees !== 0; + const changeStatusLabel = `pan changed ${panChanged ? 'yes' : 'no'}, pinch changed ${ + pinchChanged ? 'yes' : 'no' + }, rotate changed ${rotateChanged ? 'yes' : 'no'}`; return ( @@ -228,8 +231,7 @@ export function GestureLab() { fling {counts.fling} - pan changed {panChanged ? 'yes' : 'no'}, pinch changed{' '} - {pinchChanged ? 'yes' : 'no'}, rotate changed {rotateChanged ? 'yes' : 'no'} + {changeStatusLabel} diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 9a50e025b..d17e004c1 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -1,5 +1,13 @@ import XCTest +private enum RunnerInterfaceOrientation { + static let unknown = 0 + static let portrait = 1 + static let portraitUpsideDown = 2 + static let landscapeRight = 3 + static let landscapeLeft = 4 +} + extension RunnerTests { struct TouchVisualizationFrame { let x: Double @@ -648,17 +656,36 @@ extension RunnerTests { let width = Double(frame.width) let height = Double(frame.height) switch interfaceOrientation { - case 3: // landscapeRight + case RunnerInterfaceOrientation.landscapeRight: return CGPoint(x: height - localY, y: localX) - case 4: // landscapeLeft + case RunnerInterfaceOrientation.landscapeLeft: return CGPoint(x: localY, y: width - localX) - case 2: // portraitUpsideDown + case RunnerInterfaceOrientation.portraitUpsideDown: return CGPoint(x: width - localX, y: height - localY) - default: // 1 portrait, 0 unknown + default: // portrait or unknown return CGPoint(x: localX, y: localY) } } + /// Rotates an interface-oriented translation vector into the same native + /// coordinate space as `nativeSynthesizedPoint`. + func nativeSynthesizedVector( + orientedDx dx: Double, + orientedDy dy: Double, + interfaceOrientation: Int + ) -> CGVector { + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeRight: + return CGVector(dx: -dy, dy: dx) + case RunnerInterfaceOrientation.landscapeLeft: + return CGVector(dx: dy, dy: -dx) + case RunnerInterfaceOrientation.portraitUpsideDown: + return CGVector(dx: -dx, dy: -dy) + default: // portrait or unknown + return CGVector(dx: dx, dy: dy) + } + } + func synthesizedDragAt( app: XCUIApplication, x: Double, @@ -983,12 +1010,15 @@ extension RunnerTests { ) -> RunnerInteractionOutcome { #if os(iOS) let target = interactionRoot(app: app) + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation) + let vector = nativeSynthesizedVector(orientedDx: dx, orientedDy: dy, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeTransform( withApplication: app, - x: x, - y: y, - dx: dx, - dy: dy, + x: Double(point.x), + y: Double(point.y), + dx: Double(vector.dx), + dy: Double(vector.dy), scale: scale, degrees: degrees, radius: transformGestureRadius(frame: target.frame, scale: scale), @@ -1127,13 +1157,14 @@ extension RunnerTests { func testNativeSynthesizedPointRotatesByInterfaceOrientation() { let portrait = CGRect(x: 0, y: 0, width: 834, height: 1210) let landscape = CGRect(x: 0, y: 0, width: 1210, height: 834) + let offsetLandscape = CGRect(x: 10, y: 20, width: 1210, height: 834) // (frame, UIInterfaceOrientation, expected native point) for a tap at (170, 268). let cases: [(CGRect, Int, CGPoint)] = [ - (portrait, 1, CGPoint(x: 170, y: 268)), // portrait - (landscape, 3, CGPoint(x: 566, y: 170)), // landscapeRight - (landscape, 4, CGPoint(x: 268, y: 1040)), // landscapeLeft - (portrait, 2, CGPoint(x: 664, y: 942)), // portraitUpsideDown - (portrait, 0, CGPoint(x: 170, y: 268)), // unknown -> identity + (portrait, RunnerInterfaceOrientation.portrait, CGPoint(x: 170, y: 268)), + (landscape, RunnerInterfaceOrientation.landscapeRight, CGPoint(x: 566, y: 170)), + (landscape, RunnerInterfaceOrientation.landscapeLeft, CGPoint(x: 268, y: 1040)), + (portrait, RunnerInterfaceOrientation.portraitUpsideDown, CGPoint(x: 664, y: 942)), + (portrait, RunnerInterfaceOrientation.unknown, CGPoint(x: 170, y: 268)), ] for (frame, orientation, expected) in cases { XCTAssertEqual( @@ -1142,5 +1173,30 @@ extension RunnerTests { "interfaceOrientation \(orientation)" ) } + XCTAssertEqual( + nativeSynthesizedPoint( + orientedX: 180, + orientedY: 288, + in: offsetLandscape, + interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft + ), + CGPoint(x: 268, y: 1040), + "non-zero frame origin is localized before rotation" + ) + } + + func testNativeSynthesizedVectorRotatesByInterfaceOrientation() { + let cases: [(Int, CGVector)] = [ + (RunnerInterfaceOrientation.portrait, CGVector(dx: 40, dy: -20)), + (RunnerInterfaceOrientation.landscapeRight, CGVector(dx: 20, dy: 40)), + (RunnerInterfaceOrientation.landscapeLeft, CGVector(dx: -20, dy: -40)), + (RunnerInterfaceOrientation.portraitUpsideDown, CGVector(dx: -40, dy: 20)), + (RunnerInterfaceOrientation.unknown, CGVector(dx: 40, dy: -20)), + ] + for (orientation, expected) in cases { + let vector = nativeSynthesizedVector(orientedDx: 40, orientedDy: -20, interfaceOrientation: orientation) + XCTAssertEqual(vector.dx, expected.dx, "dx interfaceOrientation \(orientation)") + XCTAssertEqual(vector.dy, expected.dy, "dy interfaceOrientation \(orientation)") + } } }