Skip to content

Commit d19aa27

Browse files
thymikeethymikee
committed
perf: speed up ios swipes and harden runner cache
1 parent 491ad7e commit d19aa27

10 files changed

Lines changed: 639 additions & 104 deletions

File tree

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ NS_ASSUME_NONNULL_BEGIN
1414
radius:(double)radius
1515
durationMs:(double)durationMs;
1616

17+
+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
18+
x:(double)x
19+
y:(double)y
20+
x2:(double)x2
21+
y2:(double)y2
22+
durationMs:(double)durationMs;
23+
1724
@end
1825

1926
NS_ASSUME_NONNULL_END

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ static id RunnerPointerPath(
4747
double durationMs,
4848
double side
4949
);
50+
static id RunnerSwipePointerPath(
51+
const RunnerXCTestEventBridge *bridge,
52+
CGPoint start,
53+
CGPoint end,
54+
double durationMs
55+
);
5056
static CGPoint RunnerPointerPointAt(
5157
double x,
5258
double y,
@@ -58,6 +64,8 @@ static CGPoint RunnerPointerPointAt(
5864
double t,
5965
double side
6066
);
67+
static CGPoint RunnerInterpolatedPoint(CGPoint start, CGPoint end, double t);
68+
static double RunnerSmoothStep(double t);
6169

6270
@implementation RunnerSynthesizedGesture
6371

@@ -87,6 +95,26 @@ + (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
8795
}
8896
}
8997

98+
+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
99+
x:(double)x
100+
y:(double)y
101+
x2:(double)x2
102+
y2:(double)y2
103+
durationMs:(double)durationMs {
104+
@try {
105+
return [self trySynthesizeSwipeWithApplication:application
106+
x:x
107+
y:y
108+
x2:x2
109+
y2:y2
110+
durationMs:durationMs];
111+
} @catch (NSException *exception) {
112+
NSString *name = exception.name ?: @"NSException";
113+
NSString *reason = exception.reason ?: @"private XCTest event synthesis failed";
114+
return [NSString stringWithFormat:@"%@: %@", name, reason];
115+
}
116+
}
117+
90118
+ (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application
91119
x:(double)x
92120
y:(double)y
@@ -151,6 +179,51 @@ + (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application
151179
return nil;
152180
}
153181

182+
+ (NSString * _Nullable)trySynthesizeSwipeWithApplication:(id)application
183+
x:(double)x
184+
y:(double)y
185+
x2:(double)x2
186+
y2:(double)y2
187+
durationMs:(double)durationMs {
188+
RunnerXCTestEventBridge bridge;
189+
NSString *missing = RunnerResolveXCTestEventBridge(application, &bridge);
190+
if (missing != nil) {
191+
return missing;
192+
}
193+
194+
NSInteger interfaceOrientation =
195+
((RunnerMsgSendInteger)objc_msgSend)(application, bridge.interfaceOrientationSelector);
196+
NSInteger targetProcessID = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.processIDSelector);
197+
if (targetProcessID <= 0) {
198+
return @"private XCTest event synthesis unavailable: could not resolve target process ID";
199+
}
200+
201+
id record = ((RunnerMsgSendInitRecord)objc_msgSend)(
202+
[bridge.recordClass alloc],
203+
bridge.initRecordSelector,
204+
@"agent-device-swipe",
205+
interfaceOrientation
206+
);
207+
if (record == nil) {
208+
return @"private XCTest event synthesis failed: could not create event record";
209+
}
210+
((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.setTargetProcessIDSelector, targetProcessID);
211+
212+
id path = RunnerSwipePointerPath(&bridge, CGPointMake(x, y), CGPointMake(x2, y2), durationMs);
213+
if (path == nil) {
214+
return @"private XCTest event synthesis failed: could not create pointer path";
215+
}
216+
((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.addPathSelector, path);
217+
218+
NSError *error = nil;
219+
BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, bridge.synthesizeSelector, &error);
220+
if (!ok) {
221+
NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false";
222+
return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail];
223+
}
224+
return nil;
225+
}
226+
154227
static NSString * _Nullable RunnerResolveXCTestEventBridge(
155228
id application,
156229
RunnerXCTestEventBridge *bridge
@@ -270,6 +343,31 @@ static id RunnerPointerPath(
270343
return path;
271344
}
272345

346+
static id RunnerSwipePointerPath(
347+
const RunnerXCTestEventBridge *bridge,
348+
CGPoint start,
349+
CGPoint end,
350+
double durationMs
351+
) {
352+
id path =
353+
((RunnerMsgSendInitPath)objc_msgSend)([bridge->pathClass alloc], bridge->initPathSelector, start, 0.0);
354+
if (path == nil) {
355+
return nil;
356+
}
357+
358+
int frameCount = MAX(3, (int)(durationMs / 16.0));
359+
NSTimeInterval durationSeconds = durationMs / 1000.0;
360+
for (int index = 1; index <= frameCount; index += 1) {
361+
double t = (double)index / (double)frameCount;
362+
CGPoint point = RunnerInterpolatedPoint(start, end, RunnerSmoothStep(t));
363+
NSTimeInterval offset = durationSeconds * t;
364+
((RunnerMsgSendPathMove)objc_msgSend)(path, bridge->moveSelector, point, offset);
365+
}
366+
367+
((RunnerMsgSendPathOffset)objc_msgSend)(path, bridge->liftSelector, durationSeconds);
368+
return path;
369+
}
370+
273371
static CGPoint RunnerPointerPointAt(
274372
double x,
275373
double y,
@@ -294,4 +392,15 @@ static CGPoint RunnerPointerPointAt(
294392
return CGPointMake(centerX + cos(angle) * radius * side, centerY + sin(angle) * radius * side);
295393
}
296394

395+
static CGPoint RunnerInterpolatedPoint(CGPoint start, CGPoint end, double t) {
396+
return CGPointMake(
397+
start.x + (end.x - start.x) * t,
398+
start.y + (end.y - start.y) * t
399+
);
400+
}
401+
402+
static double RunnerSmoothStep(double t) {
403+
return t * t * (3.0 - 2.0 * t);
404+
}
405+
297406
@end

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ extension RunnerTests {
1313
return (gestureStartUptimeMs, currentUptimeMs())
1414
}
1515

16+
private func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
17+
min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120)
18+
}
19+
1620
func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
1721
switch outcome {
1822
case .performed:
@@ -495,7 +499,6 @@ extension RunnerTests {
495499
guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
496500
return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
497501
}
498-
let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
499502
let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
500503
let dragFrame = resolvedDragVisualizationFrame(
501504
app: activeApp,
@@ -504,6 +507,25 @@ extension RunnerTests {
504507
x2: dragPoints.x2,
505508
y2: dragPoints.y2
506509
)
510+
if command.synthesized == true {
511+
let durationMs = min(max(command.durationMs ?? 250, 16), 10000)
512+
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
513+
synthesizedDragAt(
514+
app: activeApp,
515+
x: dragPoints.x,
516+
y: dragPoints.y,
517+
x2: dragPoints.x2,
518+
y2: dragPoints.y2,
519+
durationMs: durationMs
520+
)
521+
}
522+
if case .performed = outcome {
523+
return gestureResponse(message: "dragged", timing: timing, frame: .drag(dragFrame))
524+
}
525+
}
526+
let holdDuration = command.synthesized == true
527+
? synthesizedSwipeFallbackHoldDuration(durationMs: command.durationMs ?? 250)
528+
: min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
507529
let (timing, outcome) = performGesture(activeApp) {
508530
dragAt(
509531
app: activeApp,

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,42 @@ extension RunnerTests {
634634
return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
635635
}
636636

637+
func synthesizedDragAt(
638+
app: XCUIApplication,
639+
x: Double,
640+
y: Double,
641+
x2: Double,
642+
y2: Double,
643+
durationMs: Double
644+
) -> RunnerInteractionOutcome {
645+
#if os(iOS)
646+
if let message = RunnerSynthesizedGesture.synthesizeSwipe(
647+
withApplication: app,
648+
x: x,
649+
y: y,
650+
x2: x2,
651+
y2: y2,
652+
durationMs: durationMs
653+
) {
654+
return .unsupported(
655+
message: message,
656+
hint: "Falling back to XCTest coordinate drag may be slower; update Xcode if this persists."
657+
)
658+
}
659+
return .performed
660+
#elseif os(tvOS)
661+
return .unsupported(
662+
message: "coordinate drag is not supported on tvOS",
663+
hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead."
664+
)
665+
#else
666+
return .unsupported(
667+
message: "coordinate drag is not supported on macOS",
668+
hint: "macOS automation has no touchscreen; use mouse-driven interactions instead."
669+
)
670+
#endif
671+
}
672+
637673
func keyboardAvoidingDragPoints(
638674
app: XCUIApplication,
639675
x: Double,

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ struct Command: Codable {
147147
let scope: String?
148148
let raw: Bool?
149149
let fullscreen: Bool?
150+
let synthesized: Bool?
150151
}
151152

152153
struct Response: Codable {

‎src/platforms/ios/__tests__/index.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,48 @@ test('iosRunnerOverrides gives fling a short default XCUITest drag hold', async
160160
});
161161
});
162162

163+
test('iosRunnerOverrides maps swipe to synthesized iOS drag duration', async () => {
164+
mockRunIosRunnerCommand.mockResolvedValue({});
165+
166+
const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, {
167+
appBundleId: 'com.example.App',
168+
});
169+
170+
await overrides.swipe(100, 200, 180, 200, 300);
171+
await overrides.swipe(100, 200, 180, 200, undefined);
172+
await overrides.pan(100, 200, 180, 200, 300);
173+
174+
assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], {
175+
command: 'drag',
176+
x: 100,
177+
y: 200,
178+
x2: 180,
179+
y2: 200,
180+
durationMs: 300,
181+
synthesized: true,
182+
appBundleId: 'com.example.App',
183+
});
184+
assert.deepEqual(mockRunIosRunnerCommand.mock.calls[1]?.[1], {
185+
command: 'drag',
186+
x: 100,
187+
y: 200,
188+
x2: 180,
189+
y2: 200,
190+
durationMs: 250,
191+
synthesized: true,
192+
appBundleId: 'com.example.App',
193+
});
194+
assert.deepEqual(mockRunIosRunnerCommand.mock.calls[2]?.[1], {
195+
command: 'drag',
196+
x: 100,
197+
y: 200,
198+
x2: 180,
199+
y2: 200,
200+
durationMs: 300,
201+
appBundleId: 'com.example.App',
202+
});
203+
});
204+
163205
test('AGENT_DEVICE_MACOS_HELPER_BIN rejects relative override paths', async () => {
164206
const previousHelperPath = process.env.AGENT_DEVICE_MACOS_HELPER_BIN;
165207
process.env.AGENT_DEVICE_MACOS_HELPER_BIN = './agent-device-macos-helper';

0 commit comments

Comments
 (0)