Skip to content

Commit cf87b48

Browse files
thymikeethymikee
committed
feat: support ios transform gesture
1 parent 93b04b2 commit cf87b48

8 files changed

Lines changed: 350 additions & 38 deletions

File tree

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
#import "RunnerObjCExceptionCatcher.h"
2+
#import "RunnerSynthesizedGesture.h"

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#import <Foundation/Foundation.h>
2+
3+
NS_ASSUME_NONNULL_BEGIN
4+
5+
@interface RunnerSynthesizedGesture : NSObject
6+
7+
+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
8+
x:(double)x
9+
y:(double)y
10+
dx:(double)dx
11+
dy:(double)dy
12+
scale:(double)scale
13+
degrees:(double)degrees
14+
radius:(double)radius
15+
durationMs:(double)durationMs;
16+
17+
@end
18+
19+
NS_ASSUME_NONNULL_END

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m‎

Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
#import "RunnerSynthesizedGesture.h"
2+
3+
#import <CoreGraphics/CoreGraphics.h>
4+
#import <math.h>
5+
#import <objc/message.h>
6+
7+
typedef NSInteger (*RunnerMsgSendInteger)(id, SEL);
8+
typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSInteger);
9+
typedef id (*RunnerMsgSendInitPath)(id, SEL, CGPoint, NSTimeInterval);
10+
typedef void (*RunnerMsgSendPathMove)(id, SEL, CGPoint, NSTimeInterval);
11+
typedef void (*RunnerMsgSendPathOffset)(id, SEL, NSTimeInterval);
12+
typedef void (*RunnerMsgSendAddPath)(id, SEL, id);
13+
typedef void (*RunnerMsgSendSetInteger)(id, SEL, NSInteger);
14+
typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **);
15+
16+
static NSString * _Nullable RunnerRequireClass(NSString *className);
17+
static NSString * _Nullable RunnerRequireSelector(Class cls, SEL selector, NSString *selectorName);
18+
static NSString * _Nullable RunnerRequireApplicationSelector(id application, SEL selector, NSString *selectorName);
19+
static id RunnerPointerPath(
20+
Class pathClass,
21+
CGPoint start,
22+
double x,
23+
double y,
24+
double dx,
25+
double dy,
26+
double scale,
27+
double degrees,
28+
double radius,
29+
double durationMs,
30+
double side
31+
);
32+
static CGPoint RunnerPointerPointAt(
33+
double x,
34+
double y,
35+
double dx,
36+
double dy,
37+
double scale,
38+
double degrees,
39+
double baseRadius,
40+
double t,
41+
double side
42+
);
43+
44+
@implementation RunnerSynthesizedGesture
45+
46+
+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
47+
x:(double)x
48+
y:(double)y
49+
dx:(double)dx
50+
dy:(double)dy
51+
scale:(double)scale
52+
degrees:(double)degrees
53+
radius:(double)radius
54+
durationMs:(double)durationMs {
55+
@try {
56+
return [self trySynthesizeTransformWithApplication:application
57+
x:x
58+
y:y
59+
dx:dx
60+
dy:dy
61+
scale:scale
62+
degrees:degrees
63+
radius:radius
64+
durationMs:durationMs];
65+
} @catch (NSException *exception) {
66+
NSString *name = exception.name ?: @"NSException";
67+
NSString *reason = exception.reason ?: @"private XCTest event synthesis failed";
68+
return [NSString stringWithFormat:@"%@: %@", name, reason];
69+
}
70+
}
71+
72+
+ (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application
73+
x:(double)x
74+
y:(double)y
75+
dx:(double)dx
76+
dy:(double)dy
77+
scale:(double)scale
78+
degrees:(double)degrees
79+
radius:(double)radius
80+
durationMs:(double)durationMs {
81+
Class recordClass = NSClassFromString(@"XCSynthesizedEventRecord");
82+
Class pathClass = NSClassFromString(@"XCPointerEventPath");
83+
84+
SEL initRecordSelector = NSSelectorFromString(@"initWithName:interfaceOrientation:");
85+
SEL addPathSelector = NSSelectorFromString(@"addPointerEventPath:");
86+
SEL setTargetProcessIDSelector = NSSelectorFromString(@"setTargetProcessID:");
87+
SEL synthesizeSelector = NSSelectorFromString(@"synthesizeWithError:");
88+
SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation");
89+
SEL processIDSelector = NSSelectorFromString(@"processID");
90+
SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
91+
SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
92+
SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
93+
94+
NSString *missing = RunnerRequireClass(@"XCSynthesizedEventRecord");
95+
if (missing != nil) {
96+
return missing;
97+
}
98+
missing = RunnerRequireClass(@"XCPointerEventPath");
99+
if (missing != nil) {
100+
return missing;
101+
}
102+
missing = RunnerRequireSelector(recordClass, initRecordSelector, @"initWithName:interfaceOrientation:");
103+
if (missing != nil) {
104+
return missing;
105+
}
106+
missing = RunnerRequireSelector(recordClass, addPathSelector, @"addPointerEventPath:");
107+
if (missing != nil) {
108+
return missing;
109+
}
110+
missing = RunnerRequireSelector(recordClass, setTargetProcessIDSelector, @"setTargetProcessID:");
111+
if (missing != nil) {
112+
return missing;
113+
}
114+
missing = RunnerRequireSelector(recordClass, synthesizeSelector, @"synthesizeWithError:");
115+
if (missing != nil) {
116+
return missing;
117+
}
118+
missing = RunnerRequireSelector(pathClass, initPathSelector, @"initForTouchAtPoint:offset:");
119+
if (missing != nil) {
120+
return missing;
121+
}
122+
missing = RunnerRequireSelector(pathClass, moveSelector, @"moveToPoint:atOffset:");
123+
if (missing != nil) {
124+
return missing;
125+
}
126+
missing = RunnerRequireSelector(pathClass, liftSelector, @"liftUpAtOffset:");
127+
if (missing != nil) {
128+
return missing;
129+
}
130+
missing = RunnerRequireApplicationSelector(application, interfaceOrientationSelector, @"interfaceOrientation");
131+
if (missing != nil) {
132+
return missing;
133+
}
134+
missing = RunnerRequireApplicationSelector(application, processIDSelector, @"processID");
135+
if (missing != nil) {
136+
return missing;
137+
}
138+
139+
NSInteger interfaceOrientation =
140+
((RunnerMsgSendInteger)objc_msgSend)(application, interfaceOrientationSelector);
141+
NSInteger targetProcessID =
142+
((RunnerMsgSendInteger)objc_msgSend)(application, processIDSelector);
143+
if (targetProcessID <= 0) {
144+
return @"private XCTest event synthesis unavailable: could not resolve target process ID";
145+
}
146+
147+
id record = ((RunnerMsgSendInitRecord)objc_msgSend)(
148+
[recordClass alloc],
149+
initRecordSelector,
150+
@"agent-device-transform",
151+
interfaceOrientation
152+
);
153+
if (record == nil) {
154+
return @"private XCTest event synthesis failed: could not create event record";
155+
}
156+
((RunnerMsgSendSetInteger)objc_msgSend)(record, setTargetProcessIDSelector, targetProcessID);
157+
158+
double sides[] = {1.0, -1.0};
159+
for (int index = 0; index < 2; index += 1) {
160+
double side = sides[index];
161+
id path = RunnerPointerPath(
162+
pathClass,
163+
RunnerPointerPointAt(x, y, dx, dy, scale, degrees, radius, 0.0, side),
164+
x,
165+
y,
166+
dx,
167+
dy,
168+
scale,
169+
degrees,
170+
radius,
171+
durationMs,
172+
side
173+
);
174+
if (path == nil) {
175+
return @"private XCTest event synthesis failed: could not create pointer path";
176+
}
177+
((RunnerMsgSendAddPath)objc_msgSend)(record, addPathSelector, path);
178+
}
179+
180+
NSError *error = nil;
181+
BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, synthesizeSelector, &error);
182+
if (!ok) {
183+
NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false";
184+
return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail];
185+
}
186+
return nil;
187+
}
188+
189+
static NSString * _Nullable RunnerRequireClass(NSString *className) {
190+
if (NSClassFromString(className) == Nil) {
191+
return [NSString stringWithFormat:@"private XCTest event synthesis unavailable: missing %@", className];
192+
}
193+
return nil;
194+
}
195+
196+
static NSString * _Nullable RunnerRequireSelector(Class cls, SEL selector, NSString *selectorName) {
197+
if (![cls instancesRespondToSelector:selector]) {
198+
return [NSString stringWithFormat:
199+
@"private XCTest event synthesis unavailable: %@ missing %@",
200+
NSStringFromClass(cls),
201+
selectorName
202+
];
203+
}
204+
return nil;
205+
}
206+
207+
static NSString * _Nullable RunnerRequireApplicationSelector(
208+
id application,
209+
SEL selector,
210+
NSString *selectorName
211+
) {
212+
if (![application respondsToSelector:selector]) {
213+
return [NSString stringWithFormat:
214+
@"private XCTest event synthesis unavailable: XCUIApplication missing %@",
215+
selectorName
216+
];
217+
}
218+
return nil;
219+
}
220+
221+
static id RunnerPointerPath(
222+
Class pathClass,
223+
CGPoint start,
224+
double x,
225+
double y,
226+
double dx,
227+
double dy,
228+
double scale,
229+
double degrees,
230+
double radius,
231+
double durationMs,
232+
double side
233+
) {
234+
SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
235+
SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
236+
SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
237+
238+
id path = ((RunnerMsgSendInitPath)objc_msgSend)([pathClass alloc], initPathSelector, start, 0.0);
239+
if (path == nil) {
240+
return nil;
241+
}
242+
243+
int frameCount = MAX(3, (int)(durationMs / 16.0));
244+
NSTimeInterval durationSeconds = durationMs / 1000.0;
245+
for (int index = 1; index <= frameCount; index += 1) {
246+
double t = (double)index / (double)frameCount;
247+
CGPoint point = RunnerPointerPointAt(x, y, dx, dy, scale, degrees, radius, t, side);
248+
NSTimeInterval offset = durationSeconds * t;
249+
((RunnerMsgSendPathMove)objc_msgSend)(path, moveSelector, point, offset);
250+
}
251+
252+
((RunnerMsgSendPathOffset)objc_msgSend)(path, liftSelector, durationSeconds);
253+
return path;
254+
}
255+
256+
static CGPoint RunnerPointerPointAt(
257+
double x,
258+
double y,
259+
double dx,
260+
double dy,
261+
double scale,
262+
double degrees,
263+
double baseRadius,
264+
double t,
265+
double side
266+
) {
267+
double centerX = x + dx * t;
268+
double centerY = y + dy * t;
269+
double startRadius = baseRadius / MAX(scale, 1.0);
270+
double endRadius = baseRadius;
271+
if (scale < 1.0) {
272+
startRadius = baseRadius;
273+
endRadius = baseRadius * scale;
274+
}
275+
double radius = startRadius + (endRadius - startRadius) * t;
276+
double angle = (-M_PI_2) + (degrees * M_PI / 180.0) * t;
277+
return CGPointMake(centerX + cos(angle) * radius * side, centerY + sin(angle) * radius * side);
278+
}
279+
280+
@end

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 20 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,35 +1281,36 @@ extension RunnerTests {
12811281
durationMs: Double
12821282
) -> RunnerInteractionOutcome {
12831283
#if os(iOS)
1284-
let holdDuration = max(0.02, min(durationMs / 1000.0, 10.0) / 3.0)
1285-
let panOutcome = performCoordinateDrag(
1286-
app: app,
1284+
let target = interactionRoot(app: app)
1285+
if let message = RunnerSynthesizedGesture.synthesizeTransform(
1286+
withApplication: app,
12871287
x: x,
12881288
y: y,
1289-
x2: x + dx,
1290-
y2: y + dy,
1291-
holdDuration: holdDuration
1292-
)
1293-
guard case .performed = panOutcome else {
1294-
return panOutcome
1295-
}
1296-
1297-
let target = gestureElement(app: app, x: x, y: y)
1298-
target.pinch(withScale: CGFloat(scale), velocity: CGFloat(scale >= 1.0 ? 1.0 : -1.0))
1299-
return performCoordinateRotateGesture(
1300-
app: app,
1289+
dx: dx,
1290+
dy: dy,
1291+
scale: scale,
13011292
degrees: degrees,
1302-
x: x,
1303-
y: y,
1304-
velocity: degrees >= 0 ? 1.0 : -1.0
1305-
)
1293+
radius: transformGestureRadius(frame: target.frame, scale: scale),
1294+
durationMs: durationMs
1295+
) {
1296+
return .unsupported(message)
1297+
}
1298+
return .performed
13061299
#elseif os(tvOS)
13071300
return .unsupported("transformGesture is not supported on tvOS")
13081301
#else
13091302
return .unsupported("transformGesture is not supported on macOS")
13101303
#endif
13111304
}
13121305

1306+
private func transformGestureRadius(frame: CGRect, scale: Double) -> Double {
1307+
let shorterSide = Double(min(frame.width, frame.height))
1308+
let frameRadius = shorterSide * 0.20
1309+
let minimumEndRadius = shorterSide * 0.08
1310+
let scaleAdjustedRadius = scale < 1.0 ? max(frameRadius, minimumEndRadius / scale) : frameRadius
1311+
return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
1312+
}
1313+
13131314
private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
13141315
#if os(tvOS)
13151316
return .unsupported("pinch is not supported on tvOS")
@@ -1361,21 +1362,6 @@ extension RunnerTests {
13611362
#endif
13621363
}
13631364

1364-
#if os(iOS)
1365-
private func gestureElement(app: XCUIApplication, x: Double, y: Double) -> XCUIElement {
1366-
let point = CGPoint(x: x, y: y)
1367-
let matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in
1368-
element.exists && element.frame.contains(point) && !element.frame.isEmpty
1369-
}
1370-
if let smallest = matches.min(by: { left, right in
1371-
(left.frame.width * left.frame.height) < (right.frame.width * right.frame.height)
1372-
}) {
1373-
return smallest
1374-
}
1375-
return interactionRoot(app: app)
1376-
}
1377-
#endif
1378-
13791365
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
13801366
let windows = app.windows.allElementsBoundByIndex
13811367
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {

‎src/utils/__tests__/args.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -962,6 +962,10 @@ test('usageForCommand resolves workflow help topic', () => {
962962
);
963963
assert.match(help, /agent-device clipboard write "some text"/);
964964
assert.match(help, /For gesture-heavy iOS simulator proof videos, prefer --hide-touches/);
965+
assert.match(
966+
help,
967+
/iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan\/scale\/rotation path/,
968+
);
965969
assert.match(help, /Android Gboard handwriting\/stylus UI can capture text/);
966970
assert.match(help, /targetInput\/actualInput details/);
967971
assert.match(help, /Do not keep retrying fill\/type against the same field/);

0 commit comments

Comments
 (0)