Skip to content

Commit f6ff245

Browse files
committed
feat: support ios transform gesture
1 parent 93b04b2 commit f6ff245

8 files changed

Lines changed: 302 additions & 38 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
#import "RunnerObjCExceptionCatcher.h"
2+
#import "RunnerSynthesizedGesture.h"
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#import <Foundation/Foundation.h>
2+
3+
NS_ASSUME_NONNULL_BEGIN
4+
5+
@interface RunnerSynthesizedGesture : NSObject
6+
7+
+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
8+
x:(double)x
9+
y:(double)y
10+
dx:(double)dx
11+
dy:(double)dy
12+
scale:(double)scale
13+
degrees:(double)degrees
14+
durationMs:(double)durationMs;
15+
16+
@end
17+
18+
NS_ASSUME_NONNULL_END
Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
#import "RunnerSynthesizedGesture.h"
2+
3+
#import <CoreGraphics/CoreGraphics.h>
4+
#import <math.h>
5+
#import <objc/message.h>
6+
7+
static const double RunnerTransformGestureRadius = 80.0;
8+
9+
typedef long long (*RunnerMsgSendLongLong)(id, SEL);
10+
typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, long long);
11+
typedef id (*RunnerMsgSendInitPath)(id, SEL, CGPoint, NSTimeInterval);
12+
typedef void (*RunnerMsgSendPathMove)(id, SEL, CGPoint, NSTimeInterval);
13+
typedef void (*RunnerMsgSendPathOffset)(id, SEL, NSTimeInterval);
14+
typedef void (*RunnerMsgSendAddPath)(id, SEL, id);
15+
typedef void (*RunnerMsgSendSetLongLong)(id, SEL, long long);
16+
typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **);
17+
18+
typedef struct RunnerPointerPair {
19+
CGPoint a;
20+
CGPoint b;
21+
} RunnerPointerPair;
22+
23+
static long long RunnerInterfaceOrientationForApplication(id application);
24+
static long long RunnerProcessIDForApplication(id application);
25+
static RunnerPointerPair RunnerPointerPairAt(
26+
double x,
27+
double y,
28+
double dx,
29+
double dy,
30+
double scale,
31+
double degrees,
32+
double baseRadius,
33+
double t
34+
);
35+
36+
@implementation RunnerSynthesizedGesture
37+
38+
+ (NSString * _Nullable)synthesizeTransformWithApplication:(id)application
39+
x:(double)x
40+
y:(double)y
41+
dx:(double)dx
42+
dy:(double)dy
43+
scale:(double)scale
44+
degrees:(double)degrees
45+
durationMs:(double)durationMs {
46+
@try {
47+
return [self trySynthesizeTransformWithApplication:application
48+
x:x
49+
y:y
50+
dx:dx
51+
dy:dy
52+
scale:scale
53+
degrees:degrees
54+
durationMs:durationMs];
55+
} @catch (NSException *exception) {
56+
NSString *name = exception.name ?: @"NSException";
57+
NSString *reason = exception.reason ?: @"private XCTest event synthesis failed";
58+
return [NSString stringWithFormat:@"%@: %@", name, reason];
59+
}
60+
}
61+
62+
+ (NSString * _Nullable)trySynthesizeTransformWithApplication:(id)application
63+
x:(double)x
64+
y:(double)y
65+
dx:(double)dx
66+
dy:(double)dy
67+
scale:(double)scale
68+
degrees:(double)degrees
69+
durationMs:(double)durationMs {
70+
Class recordClass = NSClassFromString(@"XCSynthesizedEventRecord");
71+
Class pathClass = NSClassFromString(@"XCPointerEventPath");
72+
if (recordClass == Nil || pathClass == Nil) {
73+
return @"private XCTest event synthesis unavailable: missing XCUIAutomation classes";
74+
}
75+
76+
SEL initRecordSelector = NSSelectorFromString(@"initWithName:interfaceOrientation:");
77+
SEL addPathSelector = NSSelectorFromString(@"addPointerEventPath:");
78+
SEL setTargetProcessIDSelector = NSSelectorFromString(@"setTargetProcessID:");
79+
SEL synthesizeSelector = NSSelectorFromString(@"synthesizeWithError:");
80+
SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
81+
SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
82+
SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
83+
if (![recordClass instancesRespondToSelector:initRecordSelector] ||
84+
![recordClass instancesRespondToSelector:addPathSelector] ||
85+
![recordClass instancesRespondToSelector:setTargetProcessIDSelector] ||
86+
![recordClass instancesRespondToSelector:synthesizeSelector] ||
87+
![pathClass instancesRespondToSelector:initPathSelector] ||
88+
![pathClass instancesRespondToSelector:moveSelector] ||
89+
![pathClass instancesRespondToSelector:liftSelector]) {
90+
return @"private XCTest event synthesis unavailable: required selectors are missing";
91+
}
92+
93+
long long interfaceOrientation = RunnerInterfaceOrientationForApplication(application);
94+
long long targetProcessID = RunnerProcessIDForApplication(application);
95+
if (targetProcessID <= 0) {
96+
return @"private XCTest event synthesis unavailable: could not resolve target process ID";
97+
}
98+
double radius = RunnerTransformGestureRadius;
99+
RunnerPointerPair start = RunnerPointerPairAt(x, y, dx, dy, scale, degrees, radius, 0.0);
100+
101+
id record = ((RunnerMsgSendInitRecord)objc_msgSend)(
102+
[recordClass alloc],
103+
initRecordSelector,
104+
@"agent-device-transform",
105+
interfaceOrientation
106+
);
107+
if (record == nil) {
108+
return @"private XCTest event synthesis failed: could not create event record";
109+
}
110+
((RunnerMsgSendSetLongLong)objc_msgSend)(record, setTargetProcessIDSelector, targetProcessID);
111+
112+
id pathA = [self pathWithClass:pathClass
113+
start:start.a
114+
x:x
115+
y:y
116+
dx:dx
117+
dy:dy
118+
scale:scale
119+
degrees:degrees
120+
radius:radius
121+
durationMs:durationMs
122+
finger:0
123+
downOffset:0.0
124+
liftOffset:durationMs / 1000.0];
125+
id pathB = [self pathWithClass:pathClass
126+
start:start.b
127+
x:x
128+
y:y
129+
dx:dx
130+
dy:dy
131+
scale:scale
132+
degrees:degrees
133+
radius:radius
134+
durationMs:durationMs
135+
finger:1
136+
downOffset:0.0
137+
liftOffset:durationMs / 1000.0];
138+
if (pathA == nil || pathB == nil) {
139+
return @"private XCTest event synthesis failed: could not create pointer paths";
140+
}
141+
142+
((RunnerMsgSendAddPath)objc_msgSend)(record, addPathSelector, pathA);
143+
((RunnerMsgSendAddPath)objc_msgSend)(record, addPathSelector, pathB);
144+
145+
NSError *error = nil;
146+
BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, synthesizeSelector, &error);
147+
if (!ok) {
148+
NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false";
149+
return [NSString stringWithFormat:@"private XCTest event synthesis failed: %@", detail];
150+
}
151+
return nil;
152+
}
153+
154+
static long long RunnerInterfaceOrientationForApplication(id application) {
155+
SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation");
156+
if (![application respondsToSelector:interfaceOrientationSelector]) {
157+
return 1;
158+
}
159+
long long interfaceOrientation =
160+
((RunnerMsgSendLongLong)objc_msgSend)(application, interfaceOrientationSelector);
161+
return interfaceOrientation > 0 ? interfaceOrientation : 1;
162+
}
163+
164+
static long long RunnerProcessIDForApplication(id application) {
165+
SEL processIDSelector = NSSelectorFromString(@"processID");
166+
if (![application respondsToSelector:processIDSelector]) {
167+
return 0;
168+
}
169+
return ((RunnerMsgSendLongLong)objc_msgSend)(application, processIDSelector);
170+
}
171+
172+
+ (id)pathWithClass:(Class)pathClass
173+
start:(CGPoint)start
174+
x:(double)x
175+
y:(double)y
176+
dx:(double)dx
177+
dy:(double)dy
178+
scale:(double)scale
179+
degrees:(double)degrees
180+
radius:(double)radius
181+
durationMs:(double)durationMs
182+
finger:(int)finger
183+
downOffset:(NSTimeInterval)downOffset
184+
liftOffset:(NSTimeInterval)liftOffset {
185+
SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
186+
SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
187+
SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
188+
189+
id path = ((RunnerMsgSendInitPath)objc_msgSend)([pathClass alloc], initPathSelector, start, downOffset);
190+
if (path == nil) {
191+
return nil;
192+
}
193+
194+
int frameCount = MAX(3, (int)lround(durationMs / 16.0));
195+
double durationSeconds = durationMs / 1000.0;
196+
for (int index = 1; index < frameCount; index += 1) {
197+
double t = (double)index / (double)frameCount;
198+
RunnerPointerPair pair = RunnerPointerPairAt(x, y, dx, dy, scale, degrees, radius, t);
199+
CGPoint point = finger == 0 ? pair.a : pair.b;
200+
NSTimeInterval offset = durationSeconds * t;
201+
((RunnerMsgSendPathMove)objc_msgSend)(path, moveSelector, point, offset);
202+
}
203+
204+
((RunnerMsgSendPathOffset)objc_msgSend)(path, liftSelector, liftOffset);
205+
return path;
206+
}
207+
208+
static RunnerPointerPair RunnerPointerPairAt(
209+
double x,
210+
double y,
211+
double dx,
212+
double dy,
213+
double scale,
214+
double degrees,
215+
double baseRadius,
216+
double t
217+
) {
218+
double centerX = x + dx * t;
219+
double centerY = y + dy * t;
220+
double startRadius = baseRadius / MAX(scale, 1.0);
221+
double endRadius = baseRadius;
222+
if (scale < 1.0) {
223+
startRadius = baseRadius;
224+
endRadius = baseRadius * scale;
225+
}
226+
double radius = startRadius + (endRadius - startRadius) * t;
227+
double angle = (-90.0 + degrees * t) * M_PI / 180.0;
228+
double offsetX = cos(angle) * radius;
229+
double offsetY = sin(angle) * radius;
230+
RunnerPointerPair pair = {
231+
.a = CGPointMake(centerX + offsetX, centerY + offsetY),
232+
.b = CGPointMake(centerX - offsetX, centerY - offsetY),
233+
};
234+
return pair;
235+
}
236+
237+
@end

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,28 +1281,28 @@ extension RunnerTests {
12811281
durationMs: Double
12821282
) -> RunnerInteractionOutcome {
12831283
#if os(iOS)
1284-
let holdDuration = max(0.02, min(durationMs / 1000.0, 10.0) / 3.0)
1285-
let panOutcome = performCoordinateDrag(
1286-
app: app,
1284+
if let message = RunnerSynthesizedGesture.synthesizeTransform(
1285+
withApplication: app,
12871286
x: x,
12881287
y: y,
1289-
x2: x + dx,
1290-
y2: y + dy,
1291-
holdDuration: holdDuration
1292-
)
1293-
guard case .performed = panOutcome else {
1294-
return panOutcome
1288+
dx: dx,
1289+
dy: dy,
1290+
scale: scale,
1291+
degrees: 0,
1292+
durationMs: durationMs
1293+
) {
1294+
return .unsupported(message)
1295+
}
1296+
if abs(degrees) > 0.001 {
1297+
return performCoordinateRotateGesture(
1298+
app: app,
1299+
degrees: degrees,
1300+
x: x + dx,
1301+
y: y + dy,
1302+
velocity: degrees >= 0 ? 1 : -1
1303+
)
12951304
}
1296-
1297-
let target = gestureElement(app: app, x: x, y: y)
1298-
target.pinch(withScale: CGFloat(scale), velocity: CGFloat(scale >= 1.0 ? 1.0 : -1.0))
1299-
return performCoordinateRotateGesture(
1300-
app: app,
1301-
degrees: degrees,
1302-
x: x,
1303-
y: y,
1304-
velocity: degrees >= 0 ? 1.0 : -1.0
1305-
)
1305+
return .performed
13061306
#elseif os(tvOS)
13071307
return .unsupported("transformGesture is not supported on tvOS")
13081308
#else
@@ -1361,21 +1361,6 @@ extension RunnerTests {
13611361
#endif
13621362
}
13631363

1364-
#if os(iOS)
1365-
private func gestureElement(app: XCUIApplication, x: Double, y: Double) -> XCUIElement {
1366-
let point = CGPoint(x: x, y: y)
1367-
let matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in
1368-
element.exists && element.frame.contains(point) && !element.frame.isEmpty
1369-
}
1370-
if let smallest = matches.min(by: { left, right in
1371-
(left.frame.width * left.frame.height) < (right.frame.width * right.frame.height)
1372-
}) {
1373-
return smallest
1374-
}
1375-
return interactionRoot(app: app)
1376-
}
1377-
#endif
1378-
13791364
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
13801365
let windows = app.windows.allElementsBoundByIndex
13811366
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {

src/utils/__tests__/args.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -962,6 +962,7 @@ test('usageForCommand resolves workflow help topic', () => {
962962
);
963963
assert.match(help, /agent-device clipboard write "some text"/);
964964
assert.match(help, /For gesture-heavy iOS simulator proof videos, prefer --hide-touches/);
965+
assert.match(help, /iOS simulator transform uses private XCTest synthesis for pan\/scale/);
965966
assert.match(help, /Android Gboard handwriting\/stylus UI can capture text/);
966967
assert.match(help, /targetInput\/actualInput details/);
967968
assert.match(help, /Do not keep retrying fill\/type against the same field/);

src/utils/command-schema.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ Command shape:
236236
Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output.
237237
If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @<ref>, @ref, @Label_Name, or @eN placeholders.
238238
Close means agent-device close. App-owned back means back; system back means back --system.
239-
Taps are press or click. Gestures use swipe, longpress, or gesture <pan|fling|pinch|rotate|transform>. Android pinch, rotate, and transform use provider-native touch injection when available, then the bundled multi-touch helper.
239+
Taps are press or click. Gestures use swipe, longpress, or gesture <pan|fling|pinch|rotate|transform>. Android pinch, rotate, and transform use provider-native touch injection when available, then the bundled multi-touch helper. iOS simulator transform uses private XCTest synthesis for pan/scale and XCTest rotation for degrees; otherwise it reports UNSUPPORTED_OPERATION.
240240
241241
Bootstrap:
242242
agent-device devices --platform ios
@@ -323,7 +323,7 @@ Navigation and gestures:
323323
agent-device gesture pinch 0.5 200 400
324324
agent-device gesture rotate 35 200 420
325325
agent-device gesture transform 200 420 80 -40 2 35 700
326-
iOS simulator transform uses XCTest gesture primitives; verify app metrics instead of assuming requested degrees map exactly to recognizer output.
326+
iOS simulator transform uses private XCTest synthesis for pan/scale and XCTest rotation for degrees; verify app metrics instead of assuming requested values map exactly to recognizer output.
327327
328328
Validation and evidence:
329329
Nearby mutation diff: agent-device diff snapshot -i.

test/skillgym/suites/agent-device-smoke-suite.ts

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1525,7 +1525,7 @@ const SKILL_GUIDANCE_CASES: Case[] = [
15251525
'Platform: Android',
15261526
'Current screen: gesture lab',
15271527
'Target center is x=200 y=420',
1528-
'Need one continuous two-finger gesture without lifting fingers',
1528+
'Need the direct transform command rather than separate gesture commands',
15291529
'Pan delta is dx=80 dy=-40',
15301530
'Zoom scale is 2',
15311531
'Rotation is 35 degrees',
@@ -1540,6 +1540,28 @@ const SKILL_GUIDANCE_CASES: Case[] = [
15401540
plannedCommand('compose-gestures'),
15411541
],
15421542
}),
1543+
makeCase({
1544+
id: 'ios-simulator-gesture-transform',
1545+
contract: [
1546+
'Platform: iOS simulator',
1547+
'Current screen: gesture lab',
1548+
'Target center is x=200 y=420',
1549+
'Need one continuous two-finger gesture without lifting fingers',
1550+
'Pan delta is dx=80 dy=-40',
1551+
'Zoom scale is 2',
1552+
'Rotation is 35 degrees',
1553+
'Duration is 700ms',
1554+
],
1555+
task: 'Plan the direct agent-device command for the combined pan, zoom, and rotate gesture.',
1556+
outputs: [plannedCommand('gesture transform'), /200\s+420\s+80\s+-40\s+2\s+35\s+700/i],
1557+
forbiddenOutputs: [
1558+
plannedCommand('gesture pan'),
1559+
plannedCommand('gesture pinch'),
1560+
plannedCommand('gesture rotate'),
1561+
plannedCommand('rotate-gesture'),
1562+
plannedCommand('swipe'),
1563+
],
1564+
}),
15431565
makeCase({
15441566
id: 'settings-animation-stabilizer',
15451567
contract: [

0 commit comments

Comments
 (0)