Skip to content

Commit 3785a17

Browse files
thymikeethymikee
authored
fix(ios): unify multi-touch gestures on two-finger synthesis + hinted unsupported errors (#645)
* fix(ios): unify multi-touch gestures on two-finger synthesis + hinted unsupported errors Make RunnerSynthesizedGesture the single iOS multi-touch engine and drop the older incompatible models: - rotateGesture now drives the two-finger XCTest synthesis path (dx=dy=0, scale=1, degrees), mirroring the pinch migration in #634. The native XCUIElement.rotate(withVelocity:) injected a single synthetic rotation that React Native's rotation recognizer did not read reliably; synthesis fixes it. velocity is ignored on iOS (kept in the wire contract for compatibility; rotation direction comes from the sign of degrees). - pinch is now synthesis on iOS and a clear UNSUPPORTED_OPERATION on tvOS/macOS. The macOS coordinate double-tap+drag heuristic (performCoordinatePinch) is removed: synthesis is iOS-only, so macOS multi-touch is reported honestly as unsupported rather than approximated. - RunnerInteractionOutcome.unsupported now carries an actionable hint, mapped to ErrorPayload.hint (#639). Every unsupported gesture/tvOS path returns a concise message plus a next-step hint (existing messages kept verbatim). Net: on iOS, pinch/rotate/transform all flow through one synthesis primitive. swipe/scroll/pan/ fling remain single-finger drags (correct, unchanged). Coverage: examples/test-app/replays/gesture-lab.ad exercises pinch + rotate against the gesture lab and asserts "pinch changed yes" / "rotate changed yes". * fix(ios): fail-fast macOS pinch + align capability/docs with synthesis-only Follow-through for removing the macOS coordinate pinch path (the runner now returns UNSUPPORTED_OPERATION for macOS pinch): reject it at admission instead of round-tripping. - capabilities.ts: pinch now matches rotate-gesture/transform-gesture (Android + iOS simulator only); macOS dropped. Removes the now-unused isMacOsOrMobileAppleSimulator helper. - capabilities.test.ts: pinch expected unsupported on macOS and tvOS. - website/docs/docs/commands.md: pinch listed for Android + iOS simulators only (removed from the macOS app-session list); documents that iOS rotate ignores the optional velocity arg (synthesis uses a fixed duration; direction comes from the sign of degrees). Addresses PR #645 review HIGH #2 and MEDIUM #3. * fix(ios): surface a hinted unsupported error for synthesis gestures at admission Removing macOS pinch from the capability matrix makes macOS pinch (and the already-excluded rotate-gesture/transform-gesture on macOS/tvOS/physical iOS) fail fast in ensureGenericCommandReady before reaching the runner. That left the runner's macOS-specific hint unreachable on the daemon path, so callers only saw the generic "<cmd> is not supported on this device". Add an optional unsupportedHint to the capability matrix and surface it at admission, so the synthesis-only gestures fail fast (no runner round-trip) AND return an actionable hint pointing to where they work (Android + iOS simulator). Applied to pinch / rotate-gesture / transform-gesture. Addresses PR #645 review (P2: route macOS pinch to the hinted failure).
1 parent eb3ca30 commit 3785a17

7 files changed

Lines changed: 157 additions & 80 deletions

File tree

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ extension RunnerTests {
1717
switch outcome {
1818
case .performed:
1919
return nil
20-
case .unsupported(let message):
20+
case .unsupported(let message, let hint):
2121
return Response(
2222
ok: false,
23-
error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message)
23+
error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
2424
)
2525
}
2626
}

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift‎

Lines changed: 68 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1604,13 +1604,51 @@ extension RunnerTests {
16041604
degrees: 0,
16051605
durationMs: 300
16061606
)
1607+
#elseif os(tvOS)
1608+
return .unsupported(
1609+
message: "pinch is not supported on tvOS",
1610+
hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)."
1611+
)
16071612
#else
1608-
return performCoordinatePinch(app: app, scale: scale, x: x, y: y)
1613+
return .unsupported(
1614+
message: "pinch is not supported on macOS",
1615+
hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)."
1616+
)
16091617
#endif
16101618
}
16111619

16121620
func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
1613-
return performCoordinateRotateGesture(app: app, degrees: degrees, x: x, y: y, velocity: velocity)
1621+
#if os(iOS)
1622+
// Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634)
1623+
// with zero translation/scale so React Native's rotation recognizer actually fires. The native
1624+
// XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture
1625+
// handler does not read reliably — the same class of problem #629/#634 fixed for pinch.
1626+
// velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract
1627+
// keeps it for compatibility and direction is carried entirely by the sign of `degrees`.
1628+
let frame = interactionRoot(app: app).frame
1629+
let centerX = x ?? Double(frame.midX)
1630+
let centerY = y ?? Double(frame.midY)
1631+
return transformGesture(
1632+
app: app,
1633+
x: centerX,
1634+
y: centerY,
1635+
dx: 0,
1636+
dy: 0,
1637+
scale: 1,
1638+
degrees: degrees,
1639+
durationMs: 300
1640+
)
1641+
#elseif os(tvOS)
1642+
return .unsupported(
1643+
message: "rotate-gesture is not supported on tvOS",
1644+
hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)."
1645+
)
1646+
#else
1647+
return .unsupported(
1648+
message: "rotate-gesture is not supported on macOS",
1649+
hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)."
1650+
)
1651+
#endif
16141652
}
16151653

16161654
func transformGesture(
@@ -1636,13 +1674,22 @@ extension RunnerTests {
16361674
radius: transformGestureRadius(frame: target.frame, scale: scale),
16371675
durationMs: durationMs
16381676
) {
1639-
return .unsupported(message)
1677+
return .unsupported(
1678+
message: message,
1679+
hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)."
1680+
)
16401681
}
16411682
return .performed
16421683
#elseif os(tvOS)
1643-
return .unsupported("transformGesture is not supported on tvOS")
1684+
return .unsupported(
1685+
message: "transformGesture is not supported on tvOS",
1686+
hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)."
1687+
)
16441688
#else
1645-
return .unsupported("transformGesture is not supported on macOS")
1689+
return .unsupported(
1690+
message: "transformGesture is not supported on macOS",
1691+
hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)."
1692+
)
16461693
#endif
16471694
}
16481695

@@ -1654,57 +1701,6 @@ extension RunnerTests {
16541701
return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
16551702
}
16561703

1657-
private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
1658-
#if os(tvOS)
1659-
return .unsupported("pinch is not supported on tvOS")
1660-
#else
1661-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
1662-
1663-
// Use double-tap + drag gesture for reliable map zoom
1664-
// Zoom in (scale > 1): tap then drag UP
1665-
// Zoom out (scale < 1): tap then drag DOWN
1666-
1667-
// Determine center point (use provided x/y or screen center)
1668-
let centerX = x.map { $0 / target.frame.width } ?? 0.5
1669-
let centerY = y.map { $0 / target.frame.height } ?? 0.5
1670-
let center = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: centerY))
1671-
1672-
// Calculate drag distance based on scale (clamped to reasonable range)
1673-
// Larger scale = more drag distance
1674-
let dragAmount: CGFloat
1675-
if scale > 1.0 {
1676-
// Zoom in: drag up (negative Y direction in normalized coords)
1677-
dragAmount = min(0.4, CGFloat(scale - 1.0) * 0.2)
1678-
} else {
1679-
// Zoom out: drag down (positive Y direction)
1680-
dragAmount = min(0.4, CGFloat(1.0 - scale) * 0.4)
1681-
}
1682-
1683-
let endY = scale > 1.0 ? (centerY - Double(dragAmount)) : (centerY + Double(dragAmount))
1684-
let endPoint = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: max(0.1, min(0.9, endY))))
1685-
1686-
// Tap first (first tap of double-tap)
1687-
center.tap()
1688-
1689-
// Immediately press and drag (second tap + drag)
1690-
center.press(forDuration: 0.05, thenDragTo: endPoint)
1691-
return .performed
1692-
#endif
1693-
}
1694-
1695-
private func performCoordinateRotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
1696-
#if os(iOS)
1697-
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
1698-
let radians = CGFloat(degrees * .pi / 180.0)
1699-
target.rotate(radians, withVelocity: CGFloat(velocity))
1700-
return .performed
1701-
#elseif os(tvOS)
1702-
return .unsupported("rotate-gesture is not supported on tvOS")
1703-
#else
1704-
return .unsupported("rotate-gesture is not supported on macOS")
1705-
#endif
1706-
}
1707-
17081704
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
17091705
let windows = app.windows.allElementsBoundByIndex
17101706
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
@@ -1715,7 +1711,10 @@ extension RunnerTests {
17151711

17161712
private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
17171713
#if os(tvOS)
1718-
return .unsupported("coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
1714+
return .unsupported(
1715+
message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
1716+
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
1717+
)
17191718
#else
17201719
interactionCoordinate(app: app, x: x, y: y).tap()
17211720
return .performed
@@ -1724,7 +1723,10 @@ extension RunnerTests {
17241723

17251724
private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
17261725
#if os(tvOS)
1727-
return .unsupported("coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
1726+
return .unsupported(
1727+
message: "coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
1728+
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
1729+
)
17281730
#else
17291731
interactionCoordinate(app: app, x: x, y: y).doubleTap()
17301732
return .performed
@@ -1733,7 +1735,10 @@ extension RunnerTests {
17331735

17341736
private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome {
17351737
#if os(tvOS)
1736-
return .unsupported("coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element")
1738+
return .unsupported(
1739+
message: "coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element",
1740+
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then long-select it."
1741+
)
17371742
#else
17381743
interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration)
17391744
return .performed
@@ -1749,7 +1754,10 @@ extension RunnerTests {
17491754
holdDuration: TimeInterval
17501755
) -> RunnerInteractionOutcome {
17511756
#if os(tvOS)
1752-
return .unsupported("coordinate drag is not supported on tvOS")
1757+
return .unsupported(
1758+
message: "coordinate drag is not supported on tvOS",
1759+
hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead."
1760+
)
17531761
#else
17541762
let start = interactionCoordinate(app: app, x: x, y: y)
17551763
let end = interactionCoordinate(app: app, x: x2, y: y2)

‎ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ import XCTest
22

33
enum RunnerInteractionOutcome {
44
case performed
5-
case unsupported(String)
5+
/// A capability/state gap, surfaced to the caller as an UNSUPPORTED_OPERATION error.
6+
/// `hint` is an optional actionable next step (mapped to ErrorPayload.hint).
7+
case unsupported(message: String, hint: String?)
68
}
79

810
enum TvRemoteButton {
@@ -85,7 +87,10 @@ extension RunnerTests {
8587
func selectFocusedTvElement(app: XCUIApplication, point: CGPoint, action: String) -> RunnerInteractionOutcome? {
8688
#if os(tvOS)
8789
guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else {
88-
return .unsupported("\(action) is supported on tvOS only when the requested point is inside the focused element")
90+
return .unsupported(
91+
message: "\(action) is supported on tvOS only when the requested point is inside the focused element",
92+
hint: "Move focus with swipe or scroll until the target is focused, then retry."
93+
)
8994
}
9095
_ = pressTvRemote(.select)
9196
return .performed
@@ -97,7 +102,10 @@ extension RunnerTests {
97102
func longSelectFocusedTvElement(app: XCUIApplication, point: CGPoint, duration: TimeInterval) -> RunnerInteractionOutcome? {
98103
#if os(tvOS)
99104
guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else {
100-
return .unsupported("long press is supported on tvOS only when the requested point is inside the focused element")
105+
return .unsupported(
106+
message: "long press is supported on tvOS only when the requested point is inside the focused element",
107+
hint: "Move focus with swipe or scroll until the target is focused, then retry."
108+
)
101109
}
102110
_ = pressTvRemote(.select, duration: duration)
103111
return .performed
@@ -108,7 +116,10 @@ extension RunnerTests {
108116

109117
private func performElementTap(_ element: XCUIElement) -> RunnerInteractionOutcome {
110118
#if os(tvOS)
111-
return .unsupported("element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
119+
return .unsupported(
120+
message: "element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
121+
hint: "Use swipe/scroll to move focus to the target, then select it; tvOS has no coordinate tap."
122+
)
112123
#else
113124
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
114125
element.tap()
@@ -118,7 +129,7 @@ extension RunnerTests {
118129
if isPostTapElementDisappearance(exceptionMessage) {
119130
return .performed
120131
}
121-
return .unsupported("element tap failed: \(exceptionMessage)")
132+
return .unsupported(message: "element tap failed: \(exceptionMessage)", hint: nil)
122133
}
123134
return .performed
124135
#endif
@@ -132,7 +143,10 @@ extension RunnerTests {
132143
private func selectFocusedTvElement(app: XCUIApplication, element: XCUIElement, action: String) -> RunnerInteractionOutcome? {
133144
#if os(tvOS)
134145
guard tvFocusedElementMatches(app: app, target: element) else {
135-
return .unsupported("\(action) is supported on tvOS only when the requested element is focused")
146+
return .unsupported(
147+
message: "\(action) is supported on tvOS only when the requested element is focused",
148+
hint: "Move focus to the target element first, then retry."
149+
)
136150
}
137151
_ = pressTvRemote(.select)
138152
return .performed

‎src/core/__tests__/capabilities.test.ts‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { test } from 'vitest';
22
import assert from 'node:assert/strict';
3-
import { isCommandSupportedOnDevice } from '../capabilities.ts';
3+
import { isCommandSupportedOnDevice, unsupportedHintForDevice } from '../capabilities.ts';
44
import type { DeviceInfo } from '../../utils/device.ts';
55

66
const iosSimulator: DeviceInfo = {
@@ -74,7 +74,8 @@ test('device capability matrix stays consistent across shared command groups', (
7474
{ device: iosSimulator, expected: true, label: 'on iOS sim' },
7575
{ device: iosDevice, expected: false, label: 'on iOS device' },
7676
{ device: androidDevice, expected: true, label: 'on Android' },
77-
{ device: macOsDevice, expected: true, label: 'on macOS' },
77+
{ device: macOsDevice, expected: false, label: 'on macOS' },
78+
{ device: tvOsSimulator, expected: false, label: 'on tvOS simulator' },
7879
],
7980
},
8081
{
@@ -233,7 +234,6 @@ test('macOS supports the Apple runner interaction core but excludes mobile-only
233234
'logs',
234235
'network',
235236
'open',
236-
'pinch',
237237
'perf',
238238
'press',
239239
'record',
@@ -255,6 +255,7 @@ test('macOS supports the Apple runner interaction core but excludes mobile-only
255255
'home',
256256
'install',
257257
'install-from-source',
258+
'pinch',
258259
'push',
259260
'reinstall',
260261
'rotate',
@@ -359,3 +360,33 @@ test('unknown commands default to supported', () => {
359360
assert.equal(isCommandSupportedOnDevice('some-future-cmd', androidDevice), true);
360361
assert.equal(isCommandSupportedOnDevice('some-future-cmd', linuxDevice), true);
361362
});
363+
364+
test('synthesis gestures carry an actionable unsupported hint at admission', () => {
365+
// macOS / tvOS / physical iOS are rejected at admission; the hint redirects to where the
366+
// two-finger synthesis path actually works, so callers do not just see a bare "not supported".
367+
for (const command of ['pinch', 'rotate-gesture', 'transform-gesture']) {
368+
assert.match(
369+
unsupportedHintForDevice(command, macOsDevice) ?? '',
370+
/multi-touch/i,
371+
`${command} macOS hint`,
372+
);
373+
assert.match(
374+
unsupportedHintForDevice(command, tvOsSimulator) ?? '',
375+
/touch/i,
376+
`${command} tvOS hint`,
377+
);
378+
assert.match(
379+
unsupportedHintForDevice(command, iosDevice) ?? '',
380+
/simulator/i,
381+
`${command} iOS device hint`,
382+
);
383+
// Where the gesture IS supported there is nothing to hint.
384+
assert.equal(
385+
unsupportedHintForDevice(command, iosSimulator),
386+
undefined,
387+
`${command} iOS sim (supported) hint`,
388+
);
389+
}
390+
// Commands without a hint hook return undefined (admission keeps its generic message).
391+
assert.equal(unsupportedHintForDevice('tap', macOsDevice), undefined);
392+
});

0 commit comments

Comments
 (0)