Skip to content

Commit 8f9c55b

Browse files
thymikeethymikee
committed
feat: support ios transform gesture
1 parent 93b04b2 commit 8f9c55b

8 files changed

Lines changed: 367 additions & 38 deletions

File tree

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
#import "RunnerObjCExceptionCatcher.h"
2+
#import "RunnerSynthesizedGesture.h"

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#import <Foundation/Foundation.h>
2+
3+
NS_ASSUME_NONNULL_BEGIN
4+
5+
@interface RunnerSynthesizedGesture : NSObject
6+
7+
+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
8+
x:(double)x
9+
y:(double)y
10+
dx:(double)dx
11+
dy:(double)dy
12+
scale:(double)scale
13+
degrees:(double)degrees
14+
radius:(double)radius
15+
durationMs:(double)durationMs;
16+
17+
@end
18+
19+
NS_ASSUME_NONNULL_END

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
#import "RunnerSynthesizedGesture.h"
2+
3+
#import <CoreGraphics/CoreGraphics.h>
4+
#import <math.h>
5+
#import <objc/message.h>
6+
7+
typedef NSInteger (*RunnerMsgSendInteger)(id, SEL);
8+
typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSInteger);
9+
typedef id (*RunnerMsgSendInitPath)(id, SEL, CGPoint, NSTimeInterval);
10+
typedef void (*RunnerMsgSendPathMove)(id, SEL, CGPoint, NSTimeInterval);
11+
typedef void (*RunnerMsgSendPathOffset)(id, SEL, NSTimeInterval);
12+
typedef void (*RunnerMsgSendAddPath)(id, SEL, id);
13+
typedef void (*RunnerMsgSendSetInteger)(id, SEL, NSInteger);
14+
typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **);
15+
16+
typedef struct {
17+
Class recordClass;
18+
Class pathClass;
19+
SEL initRecordSelector;
20+
SEL addPathSelector;
21+
SEL setTargetProcessIDSelector;
22+
SEL synthesizeSelector;
23+
SEL interfaceOrientationSelector;
24+
SEL processIDSelector;
25+
SEL initPathSelector;
26+
SEL moveSelector;
27+
SEL liftSelector;
28+
} RunnerXCTestEventBridge;
29+
30+
static NSString * _Nullable RunnerResolveXCTestEventBridge(
31+
id application,
32+
RunnerXCTestEventBridge *bridge
33+
);
34+
static NSString * _Nullable RunnerRequireClass(Class cls, NSString *className);
35+
static NSString * _Nullable RunnerRequireSelector(Class cls, SEL selector, NSString *selectorName);
36+
static NSString * _Nullable RunnerRequireApplicationSelector(id application, SEL selector, NSString *selectorName);
37+
static id RunnerPointerPath(
38+
const RunnerXCTestEventBridge *bridge,
39+
CGPoint start,
40+
double x,
41+
double y,
42+
double dx,
43+
double dy,
44+
double scale,
45+
double degrees,
46+
double radius,
47+
double durationMs,
48+
double side
49+
);
50+
static CGPoint RunnerPointerPointAt(
51+
double x,
52+
double y,
53+
double dx,
54+
double dy,
55+
double scale,
56+
double degrees,
57+
double baseRadius,
58+
double t,
59+
double side
60+
);
61+
62+
@implementation RunnerSynthesizedGesture
63+
64+
+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
65+
x:(double)x
66+
y:(double)y
67+
dx:(double)dx
68+
dy:(double)dy
69+
scale:(double)scale
70+
degrees:(double)degrees
71+
radius:(double)radius
72+
durationMs:(double)durationMs {
73+
@try {
74+
return [self trySynthesizeTransformWithApplication:application
75+
x:x
76+
y:y
77+
dx:dx
78+
dy:dy
79+
scale:scale
80+
degrees:degrees
81+
radius:radius
82+
durationMs:durationMs];
83+
} @catch (NSException *exception) {
84+
NSString *name = exception.name ?: @"NSException";
85+
NSString *reason = exception.reason ?: @"private XCTest event synthesis failed";
86+
return [NSString stringWithFormat:@"%@: %@", name, reason];
87+
}
88+
}
89+
90+
+ (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application
91+
x:(double)x
92+
y:(double)y
93+
dx:(double)dx
94+
dy:(double)dy
95+
scale:(double)scale
96+
degrees:(double)degrees
97+
radius:(double)radius
98+
durationMs:(double)durationMs {
99+
RunnerXCTestEventBridge bridge;
100+
NSString *missing = RunnerResolveXCTestEventBridge(application, &bridge);
101+
if (missing != nil) {
102+
return missing;
103+
}
104+
105+
NSInteger interfaceOrientation =
106+
((RunnerMsgSendInteger)objc_msgSend)(application, bridge.interfaceOrientationSelector);
107+
NSInteger targetProcessID = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.processIDSelector);
108+
if (targetProcessID <= 0) {
109+
return @"private XCTest event synthesis unavailable: could not resolve target process ID";
110+
}
111+
112+
id record = ((RunnerMsgSendInitRecord)objc_msgSend)(
113+
[bridge.recordClass alloc],
114+
bridge.initRecordSelector,
115+
@"agent-device-transform",
116+
interfaceOrientation
117+
);
118+
if (record == nil) {
119+
return @"private XCTest event synthesis failed: could not create event record";
120+
}
121+
((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.setTargetProcessIDSelector, targetProcessID);
122+
123+
double sides[] = {1.0, -1.0};
124+
for (int index = 0; index < 2; index += 1) {
125+
double side = sides[index];
126+
id path = RunnerPointerPath(
127+
&bridge,
128+
RunnerPointerPointAt(x, y, dx, dy, scale, degrees, radius, 0.0, side),
129+
x,
130+
y,
131+
dx,
132+
dy,
133+
scale,
134+
degrees,
135+
radius,
136+
durationMs,
137+
side
138+
);
139+
if (path == nil) {
140+
return @"private XCTest event synthesis failed: could not create pointer path";
141+
}
142+
((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.addPathSelector, path);
143+
}
144+
145+
NSError *error = nil;
146+
BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, bridge.synthesizeSelector, &error);
147+
if (!ok) {
148+
NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false";
149+
return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail];
150+
}
151+
return nil;
152+
}
153+
154+
static NSString * _Nullable RunnerResolveXCTestEventBridge(
155+
id application,
156+
RunnerXCTestEventBridge *bridge
157+
) {
158+
Class recordClass = NSClassFromString(@"XCSynthesizedEventRecord");
159+
Class pathClass = NSClassFromString(@"XCPointerEventPath");
160+
SEL initRecordSelector = NSSelectorFromString(@"initWithName:interfaceOrientation:");
161+
SEL addPathSelector = NSSelectorFromString(@"addPointerEventPath:");
162+
SEL setTargetProcessIDSelector = NSSelectorFromString(@"setTargetProcessID:");
163+
SEL synthesizeSelector = NSSelectorFromString(@"synthesizeWithError:");
164+
SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation");
165+
SEL processIDSelector = NSSelectorFromString(@"processID");
166+
SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
167+
SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
168+
SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
169+
170+
NSString *missing = RunnerRequireClass(recordClass, @"XCSynthesizedEventRecord");
171+
if (missing != nil) return missing;
172+
missing = RunnerRequireClass(pathClass, @"XCPointerEventPath");
173+
if (missing != nil) return missing;
174+
missing = RunnerRequireSelector(recordClass, initRecordSelector, @"initWithName:interfaceOrientation:");
175+
if (missing != nil) return missing;
176+
missing = RunnerRequireSelector(recordClass, addPathSelector, @"addPointerEventPath:");
177+
if (missing != nil) return missing;
178+
missing = RunnerRequireSelector(recordClass, setTargetProcessIDSelector, @"setTargetProcessID:");
179+
if (missing != nil) return missing;
180+
missing = RunnerRequireSelector(recordClass, synthesizeSelector, @"synthesizeWithError:");
181+
if (missing != nil) return missing;
182+
missing = RunnerRequireSelector(pathClass, initPathSelector, @"initForTouchAtPoint:offset:");
183+
if (missing != nil) return missing;
184+
missing = RunnerRequireSelector(pathClass, moveSelector, @"moveToPoint:atOffset:");
185+
if (missing != nil) return missing;
186+
missing = RunnerRequireSelector(pathClass, liftSelector, @"liftUpAtOffset:");
187+
if (missing != nil) return missing;
188+
missing = RunnerRequireApplicationSelector(application, interfaceOrientationSelector, @"interfaceOrientation");
189+
if (missing != nil) return missing;
190+
missing = RunnerRequireApplicationSelector(application, processIDSelector, @"processID");
191+
if (missing != nil) return missing;
192+
193+
*bridge = (RunnerXCTestEventBridge){
194+
.recordClass = recordClass,
195+
.pathClass = pathClass,
196+
.initRecordSelector = initRecordSelector,
197+
.addPathSelector = addPathSelector,
198+
.setTargetProcessIDSelector = setTargetProcessIDSelector,
199+
.synthesizeSelector = synthesizeSelector,
200+
.interfaceOrientationSelector = interfaceOrientationSelector,
201+
.processIDSelector = processIDSelector,
202+
.initPathSelector = initPathSelector,
203+
.moveSelector = moveSelector,
204+
.liftSelector = liftSelector,
205+
};
206+
return nil;
207+
}
208+
209+
static NSString * _Nullable RunnerRequireClass(Class cls, NSString *className) {
210+
if (cls == Nil) {
211+
return [NSString stringWithFormat:@"private XCTest event synthesis unavailable: missing %@", className];
212+
}
213+
return nil;
214+
}
215+
216+
static NSString * _Nullable RunnerRequireSelector(Class cls, SEL selector, NSString *selectorName) {
217+
if (![cls instancesRespondToSelector:selector]) {
218+
return [NSString stringWithFormat:
219+
@"private XCTest event synthesis unavailable: %@ missing %@",
220+
NSStringFromClass(cls),
221+
selectorName
222+
];
223+
}
224+
return nil;
225+
}
226+
227+
static NSString * _Nullable RunnerRequireApplicationSelector(
228+
id application,
229+
SEL selector,
230+
NSString *selectorName
231+
) {
232+
if (![application respondsToSelector:selector]) {
233+
return [NSString stringWithFormat:
234+
@"private XCTest event synthesis unavailable: XCUIApplication missing %@",
235+
selectorName
236+
];
237+
}
238+
return nil;
239+
}
240+
241+
static id RunnerPointerPath(
242+
const RunnerXCTestEventBridge *bridge,
243+
CGPoint start,
244+
double x,
245+
double y,
246+
double dx,
247+
double dy,
248+
double scale,
249+
double degrees,
250+
double radius,
251+
double durationMs,
252+
double side
253+
) {
254+
id path =
255+
((RunnerMsgSendInitPath)objc_msgSend)([bridge->pathClass alloc], bridge->initPathSelector, start, 0.0);
256+
if (path == nil) {
257+
return nil;
258+
}
259+
260+
int frameCount = MAX(3, (int)(durationMs / 16.0));
261+
NSTimeInterval durationSeconds = durationMs / 1000.0;
262+
for (int index = 1; index <= frameCount; index += 1) {
263+
double t = (double)index / (double)frameCount;
264+
CGPoint point = RunnerPointerPointAt(x, y, dx, dy, scale, degrees, radius, t, side);
265+
NSTimeInterval offset = durationSeconds * t;
266+
((RunnerMsgSendPathMove)objc_msgSend)(path, bridge->moveSelector, point, offset);
267+
}
268+
269+
((RunnerMsgSendPathOffset)objc_msgSend)(path, bridge->liftSelector, durationSeconds);
270+
return path;
271+
}
272+
273+
static CGPoint RunnerPointerPointAt(
274+
double x,
275+
double y,
276+
double dx,
277+
double dy,
278+
double scale,
279+
double degrees,
280+
double baseRadius,
281+
double t,
282+
double side
283+
) {
284+
double centerX = x + dx * t;
285+
double centerY = y + dy * t;
286+
double startRadius = baseRadius / MAX(scale, 1.0);
287+
double endRadius = baseRadius;
288+
if (scale < 1.0) {
289+
startRadius = baseRadius;
290+
endRadius = baseRadius * scale;
291+
}
292+
double radius = startRadius + (endRadius - startRadius) * t;
293+
double angle = (-M_PI_2) + (degrees * M_PI / 180.0) * t;
294+
return CGPointMake(centerX + cos(angle) * radius * side, centerY + sin(angle) * radius * side);
295+
}
296+
297+
@end

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 20 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,35 +1281,36 @@ extension RunnerTests {
12811281
durationMs: Double
12821282
) -> RunnerInteractionOutcome {
12831283
#if os(iOS)
1284-
let holdDuration = max(0.02, min(durationMs / 1000.0, 10.0) / 3.0)
1285-
let panOutcome = performCoordinateDrag(
1286-
app: app,
1284+
let target = interactionRoot(app: app)
1285+
if let message = RunnerSynthesizedGesture.synthesizeTransform(
1286+
withApplication: app,
12871287
x: x,
12881288
y: y,
1289-
x2: x + dx,
1290-
y2: y + dy,
1291-
holdDuration: holdDuration
1292-
)
1293-
guard case .performed = panOutcome else {
1294-
return panOutcome
1295-
}
1296-
1297-
let target = gestureElement(app: app, x: x, y: y)
1298-
target.pinch(withScale: CGFloat(scale), velocity: CGFloat(scale >= 1.0 ? 1.0 : -1.0))
1299-
return performCoordinateRotateGesture(
1300-
app: app,
1289+
dx: dx,
1290+
dy: dy,
1291+
scale: scale,
13011292
degrees: degrees,
1302-
x: x,
1303-
y: y,
1304-
velocity: degrees >= 0 ? 1.0 : -1.0
1305-
)
1293+
radius: transformGestureRadius(frame: target.frame, scale: scale),
1294+
durationMs: durationMs
1295+
) {
1296+
return .unsupported(message)
1297+
}
1298+
return .performed
13061299
#elseif os(tvOS)
13071300
return .unsupported("transformGesture is not supported on tvOS")
13081301
#else
13091302
return .unsupported("transformGesture is not supported on macOS")
13101303
#endif
13111304
}
13121305

1306+
private func transformGestureRadius(frame: CGRect, scale: Double) -> Double {
1307+
let shorterSide = Double(min(frame.width, frame.height))
1308+
let frameRadius = shorterSide * 0.20
1309+
let minimumEndRadius = shorterSide * 0.08
1310+
let scaleAdjustedRadius = scale < 1.0 ? max(frameRadius, minimumEndRadius / scale) : frameRadius
1311+
return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
1312+
}
1313+
13131314
private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
13141315
#if os(tvOS)
13151316
return .unsupported("pinch is not supported on tvOS")
@@ -1361,21 +1362,6 @@ extension RunnerTests {
13611362
#endif
13621363
}
13631364

1364-
#if os(iOS)
1365-
private func gestureElement(app: XCUIApplication, x: Double, y: Double) -> XCUIElement {
1366-
let point = CGPoint(x: x, y: y)
1367-
let matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in
1368-
element.exists && element.frame.contains(point) && !element.frame.isEmpty
1369-
}
1370-
if let smallest = matches.min(by: { left, right in
1371-
(left.frame.width * left.frame.height) < (right.frame.width * right.frame.height)
1372-
}) {
1373-
return smallest
1374-
}
1375-
return interactionRoot(app: app)
1376-
}
1377-
#endif
1378-
13791365
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
13801366
let windows = app.windows.allElementsBoundByIndex
13811367
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {

0 commit comments

Comments
 (0)