From f5e324082d35126f002ff9baab132917ee1ad881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 1 Jun 2026 17:02:20 +0200 Subject: [PATCH 1/3] fix(ios): unify multi-touch gestures on two-finger synthesis + hinted unsupported errors Make RunnerSynthesizedGesture the single iOS multi-touch engine and drop the older incompatible models: - rotateGesture now drives the two-finger XCTest synthesis path (dx=dy=0, scale=1, degrees), mirroring the pinch migration in #634. The native XCUIElement.rotate(withVelocity:) injected a single synthetic rotation that React Native's rotation recognizer did not read reliably; synthesis fixes it. velocity is ignored on iOS (kept in the wire contract for compatibility; rotation direction comes from the sign of degrees). - pinch is now synthesis on iOS and a clear UNSUPPORTED_OPERATION on tvOS/macOS. The macOS coordinate double-tap+drag heuristic (performCoordinatePinch) is removed: synthesis is iOS-only, so macOS multi-touch is reported honestly as unsupported rather than approximated. - RunnerInteractionOutcome.unsupported now carries an actionable hint, mapped to ErrorPayload.hint (#639). Every unsupported gesture/tvOS path returns a concise message plus a next-step hint (existing messages kept verbatim). Net: on iOS, pinch/rotate/transform all flow through one synthesis primitive. swipe/scroll/pan/ fling remain single-finger drags (correct, unchanged). Coverage: examples/test-app/replays/gesture-lab.ad exercises pinch + rotate against the gesture lab and asserts "pinch changed yes" / "rotate changed yes". --- .../RunnerTests+CommandExecution.swift | 4 +- .../RunnerTests+Interaction.swift | 128 ++++++++++-------- .../RunnerTests+TvRemote.swift | 26 +++- 3 files changed, 90 insertions(+), 68 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 0fe2de6c8..0541a82e4 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -17,10 +17,10 @@ extension RunnerTests { switch outcome { case .performed: return nil - case .unsupported(let message): + case .unsupported(let message, let hint): return Response( ok: false, - error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message) + error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) ) } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index a7a8cfeca..2febb3d1c 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -1604,13 +1604,51 @@ extension RunnerTests { degrees: 0, durationMs: 300 ) +#elseif os(tvOS) + return .unsupported( + message: "pinch is not supported on tvOS", + hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)." + ) #else - return performCoordinatePinch(app: app, scale: scale, x: x, y: y) + return .unsupported( + message: "pinch is not supported on macOS", + hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)." + ) #endif } func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome { - return performCoordinateRotateGesture(app: app, degrees: degrees, x: x, y: y, velocity: velocity) +#if os(iOS) + // Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634) + // with zero translation/scale so React Native's rotation recognizer actually fires. The native + // XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture + // handler does not read reliably — the same class of problem #629/#634 fixed for pinch. + // velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract + // keeps it for compatibility and direction is carried entirely by the sign of `degrees`. + let frame = interactionRoot(app: app).frame + let centerX = x ?? Double(frame.midX) + let centerY = y ?? Double(frame.midY) + return transformGesture( + app: app, + x: centerX, + y: centerY, + dx: 0, + dy: 0, + scale: 1, + degrees: degrees, + durationMs: 300 + ) +#elseif os(tvOS) + return .unsupported( + message: "rotate-gesture is not supported on tvOS", + hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)." + ) +#else + return .unsupported( + message: "rotate-gesture is not supported on macOS", + hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)." + ) +#endif } func transformGesture( @@ -1636,13 +1674,22 @@ extension RunnerTests { radius: transformGestureRadius(frame: target.frame, scale: scale), durationMs: durationMs ) { - return .unsupported(message) + return .unsupported( + message: message, + hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)." + ) } return .performed #elseif os(tvOS) - return .unsupported("transformGesture is not supported on tvOS") + return .unsupported( + message: "transformGesture is not supported on tvOS", + hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)." + ) #else - return .unsupported("transformGesture is not supported on macOS") + return .unsupported( + message: "transformGesture is not supported on macOS", + hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)." + ) #endif } @@ -1654,57 +1701,6 @@ extension RunnerTests { return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35) } - private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome { -#if os(tvOS) - return .unsupported("pinch is not supported on tvOS") -#else - let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app - - // Use double-tap + drag gesture for reliable map zoom - // Zoom in (scale > 1): tap then drag UP - // Zoom out (scale < 1): tap then drag DOWN - - // Determine center point (use provided x/y or screen center) - let centerX = x.map { $0 / target.frame.width } ?? 0.5 - let centerY = y.map { $0 / target.frame.height } ?? 0.5 - let center = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: centerY)) - - // Calculate drag distance based on scale (clamped to reasonable range) - // Larger scale = more drag distance - let dragAmount: CGFloat - if scale > 1.0 { - // Zoom in: drag up (negative Y direction in normalized coords) - dragAmount = min(0.4, CGFloat(scale - 1.0) * 0.2) - } else { - // Zoom out: drag down (positive Y direction) - dragAmount = min(0.4, CGFloat(1.0 - scale) * 0.4) - } - - let endY = scale > 1.0 ? (centerY - Double(dragAmount)) : (centerY + Double(dragAmount)) - let endPoint = target.coordinate(withNormalizedOffset: CGVector(dx: centerX, dy: max(0.1, min(0.9, endY)))) - - // Tap first (first tap of double-tap) - center.tap() - - // Immediately press and drag (second tap + drag) - center.press(forDuration: 0.05, thenDragTo: endPoint) - return .performed -#endif - } - - private func performCoordinateRotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome { -#if os(iOS) - let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app - let radians = CGFloat(degrees * .pi / 180.0) - target.rotate(radians, withVelocity: CGFloat(velocity)) - return .performed -#elseif os(tvOS) - return .unsupported("rotate-gesture is not supported on tvOS") -#else - return .unsupported("rotate-gesture is not supported on macOS") -#endif - } - private func interactionRoot(app: XCUIApplication) -> XCUIElement { let windows = app.windows.allElementsBoundByIndex if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) { @@ -1715,7 +1711,10 @@ extension RunnerTests { private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { #if os(tvOS) - return .unsupported("coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element") + return .unsupported( + message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", + hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." + ) #else interactionCoordinate(app: app, x: x, y: y).tap() return .performed @@ -1724,7 +1723,10 @@ extension RunnerTests { private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { #if os(tvOS) - return .unsupported("coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element") + return .unsupported( + message: "coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", + hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." + ) #else interactionCoordinate(app: app, x: x, y: y).doubleTap() return .performed @@ -1733,7 +1735,10 @@ extension RunnerTests { private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome { #if os(tvOS) - return .unsupported("coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element") + return .unsupported( + message: "coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element", + hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then long-select it." + ) #else interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration) return .performed @@ -1749,7 +1754,10 @@ extension RunnerTests { holdDuration: TimeInterval ) -> RunnerInteractionOutcome { #if os(tvOS) - return .unsupported("coordinate drag is not supported on tvOS") + return .unsupported( + message: "coordinate drag is not supported on tvOS", + hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead." + ) #else let start = interactionCoordinate(app: app, x: x, y: y) let end = interactionCoordinate(app: app, x: x2, y: y2) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift index 2c4029c49..9e1edaed5 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift @@ -2,7 +2,9 @@ import XCTest enum RunnerInteractionOutcome { case performed - case unsupported(String) + /// A capability/state gap, surfaced to the caller as an UNSUPPORTED_OPERATION error. + /// `hint` is an optional actionable next step (mapped to ErrorPayload.hint). + case unsupported(message: String, hint: String?) } enum TvRemoteButton { @@ -85,7 +87,10 @@ extension RunnerTests { func selectFocusedTvElement(app: XCUIApplication, point: CGPoint, action: String) -> RunnerInteractionOutcome? { #if os(tvOS) guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else { - return .unsupported("\(action) is supported on tvOS only when the requested point is inside the focused element") + return .unsupported( + message: "\(action) is supported on tvOS only when the requested point is inside the focused element", + hint: "Move focus with swipe or scroll until the target is focused, then retry." + ) } _ = pressTvRemote(.select) return .performed @@ -97,7 +102,10 @@ extension RunnerTests { func longSelectFocusedTvElement(app: XCUIApplication, point: CGPoint, duration: TimeInterval) -> RunnerInteractionOutcome? { #if os(tvOS) guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else { - return .unsupported("long press is supported on tvOS only when the requested point is inside the focused element") + return .unsupported( + message: "long press is supported on tvOS only when the requested point is inside the focused element", + hint: "Move focus with swipe or scroll until the target is focused, then retry." + ) } _ = pressTvRemote(.select, duration: duration) return .performed @@ -108,7 +116,10 @@ extension RunnerTests { private func performElementTap(_ element: XCUIElement) -> RunnerInteractionOutcome { #if os(tvOS) - return .unsupported("element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element") + return .unsupported( + message: "element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", + hint: "Use swipe/scroll to move focus to the target, then select it; tvOS has no coordinate tap." + ) #else let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ element.tap() @@ -118,7 +129,7 @@ extension RunnerTests { if isPostTapElementDisappearance(exceptionMessage) { return .performed } - return .unsupported("element tap failed: \(exceptionMessage)") + return .unsupported(message: "element tap failed: \(exceptionMessage)", hint: nil) } return .performed #endif @@ -132,7 +143,10 @@ extension RunnerTests { private func selectFocusedTvElement(app: XCUIApplication, element: XCUIElement, action: String) -> RunnerInteractionOutcome? { #if os(tvOS) guard tvFocusedElementMatches(app: app, target: element) else { - return .unsupported("\(action) is supported on tvOS only when the requested element is focused") + return .unsupported( + message: "\(action) is supported on tvOS only when the requested element is focused", + hint: "Move focus to the target element first, then retry." + ) } _ = pressTvRemote(.select) return .performed From 7db77267cd796d59d488d84a05f912779b580fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 1 Jun 2026 18:43:37 +0200 Subject: [PATCH 2/3] fix(ios): fail-fast macOS pinch + align capability/docs with synthesis-only Follow-through for removing the macOS coordinate pinch path (the runner now returns UNSUPPORTED_OPERATION for macOS pinch): reject it at admission instead of round-tripping. - capabilities.ts: pinch now matches rotate-gesture/transform-gesture (Android + iOS simulator only); macOS dropped. Removes the now-unused isMacOsOrMobileAppleSimulator helper. - capabilities.test.ts: pinch expected unsupported on macOS and tvOS. - website/docs/docs/commands.md: pinch listed for Android + iOS simulators only (removed from the macOS app-session list); documents that iOS rotate ignores the optional velocity arg (synthesis uses a fixed duration; direction comes from the sign of degrees). Addresses PR #645 review HIGH #2 and MEDIUM #3. --- src/core/__tests__/capabilities.test.ts | 5 +++-- src/core/capabilities.ts | 10 +++++----- website/docs/docs/commands.md | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index ca140a59c..8ef0e26b6 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -74,7 +74,8 @@ test('device capability matrix stays consistent across shared command groups', ( { device: iosSimulator, expected: true, label: 'on iOS sim' }, { device: iosDevice, expected: false, label: 'on iOS device' }, { device: androidDevice, expected: true, label: 'on Android' }, - { device: macOsDevice, expected: true, label: 'on macOS' }, + { device: macOsDevice, expected: false, label: 'on macOS' }, + { device: tvOsSimulator, expected: false, label: 'on tvOS simulator' }, ], }, { @@ -233,7 +234,6 @@ test('macOS supports the Apple runner interaction core but excludes mobile-only 'logs', 'network', 'open', - 'pinch', 'perf', 'press', 'record', @@ -255,6 +255,7 @@ test('macOS supports the Apple runner interaction core but excludes mobile-only 'home', 'install', 'install-from-source', + 'pinch', 'push', 'reinstall', 'rotate', diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 29c57d4b1..c67ca46e9 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -17,8 +17,6 @@ export type CommandCapability = { const isNotMacOs = (device: DeviceInfo): boolean => device.platform !== 'macos'; const isMacOsOrAppleSimulator = (device: DeviceInfo): boolean => device.platform === 'macos' || device.kind === 'simulator'; -const isMacOsOrMobileAppleSimulator = (device: DeviceInfo): boolean => - device.platform === 'macos' || (device.kind === 'simulator' && device.target !== 'tv'); const isIosMobileSimulator = (device: DeviceInfo): boolean => device.platform === 'ios' && device.kind === 'simulator' && device.target !== 'tv'; @@ -55,12 +53,14 @@ const COMMAND_CAPABILITY_MATRIX: Record = { supports: (device) => device.platform === 'android' || isMacOsOrAppleSimulator(device), }, pinch: { - // macOS desktop targets report kind=device, so this stays enabled here and the - // supports() guard excludes iOS physical devices. apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, - supports: (device) => device.platform === 'android' || isMacOsOrMobileAppleSimulator(device), + // iOS-simulator-only (plus Android): pinch is driven by the two-finger XCTest synthesis + // path (RunnerSynthesizedGesture), which is iOS-only. macOS has no multi-touch synthesis, so + // it is excluded and fails fast at admission rather than round-tripping to an unsupported + // runner. Matches rotate-gesture / transform-gesture. + supports: (device) => device.platform === 'android' || isIosMobileSimulator(device), }, 'rotate-gesture': { apple: { simulator: true, device: true }, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 0ffadfa2a..7f81ed7d4 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -158,7 +158,7 @@ agent-device snapshot -i --platform apple --target desktop - Status-item apps often expose little or no useful UI through the default macOS `app` surface. Prefer `--surface menubar` for discovery when the app lives in the top menu bar. - Use `frontmost-app`, `desktop`, and `menubar` mainly for `snapshot`, `get`, `is`, and `wait`. - If you inspect with `desktop` or `menubar` and then need to click or fill inside one app, open that app in a normal `app` session. -- macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `alert`, `gesture pinch` in app sessions, `settings appearance`, and `settings permission `. +- macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `alert`, `settings appearance`, and `settings permission `. - In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop. - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. @@ -302,8 +302,8 @@ done ``` `longpress` is supported on iOS and Android. -`gesture pinch` is supported on Android, Apple simulators, and macOS app sessions. -`gesture rotate` is supported on Android and iOS simulator app sessions. Use `rotate` for device orientation. +`gesture pinch` is supported on Android and iOS simulator app sessions. +`gesture rotate` is supported on Android and iOS simulator app sessions. Use `rotate` for device orientation. On iOS the optional `velocity` argument is ignored — rotation is synthesized over a fixed duration and direction is taken from the sign of `degrees`. `gesture transform` is supported on Android and iOS simulator app sessions. ## Find (semantic) From 567e4e861672c2c8080165c4c30e2d564fb04395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 1 Jun 2026 19:09:46 +0200 Subject: [PATCH 3/3] fix(ios): surface a hinted unsupported error for synthesis gestures at admission Removing macOS pinch from the capability matrix makes macOS pinch (and the already-excluded rotate-gesture/transform-gesture on macOS/tvOS/physical iOS) fail fast in ensureGenericCommandReady before reaching the runner. That left the runner's macOS-specific hint unreachable on the daemon path, so callers only saw the generic " is not supported on this device". Add an optional unsupportedHint to the capability matrix and surface it at admission, so the synthesis-only gestures fail fast (no runner round-trip) AND return an actionable hint pointing to where they work (Android + iOS simulator). Applied to pinch / rotate-gesture / transform-gesture. Addresses PR #645 review (P2: route macOS pinch to the hinted failure). --- src/core/__tests__/capabilities.test.ts | 32 ++++++++++++++++++++++++- src/core/capabilities.ts | 22 +++++++++++++++++ src/daemon/request-generic-dispatch.ts | 4 +++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index 8ef0e26b6..3b3604ad5 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { isCommandSupportedOnDevice } from '../capabilities.ts'; +import { isCommandSupportedOnDevice, unsupportedHintForDevice } from '../capabilities.ts'; import type { DeviceInfo } from '../../utils/device.ts'; const iosSimulator: DeviceInfo = { @@ -360,3 +360,33 @@ test('unknown commands default to supported', () => { assert.equal(isCommandSupportedOnDevice('some-future-cmd', androidDevice), true); assert.equal(isCommandSupportedOnDevice('some-future-cmd', linuxDevice), true); }); + +test('synthesis gestures carry an actionable unsupported hint at admission', () => { + // macOS / tvOS / physical iOS are rejected at admission; the hint redirects to where the + // two-finger synthesis path actually works, so callers do not just see a bare "not supported". + for (const command of ['pinch', 'rotate-gesture', 'transform-gesture']) { + assert.match( + unsupportedHintForDevice(command, macOsDevice) ?? '', + /multi-touch/i, + `${command} macOS hint`, + ); + assert.match( + unsupportedHintForDevice(command, tvOsSimulator) ?? '', + /touch/i, + `${command} tvOS hint`, + ); + assert.match( + unsupportedHintForDevice(command, iosDevice) ?? '', + /simulator/i, + `${command} iOS device hint`, + ); + // Where the gesture IS supported there is nothing to hint. + assert.equal( + unsupportedHintForDevice(command, iosSimulator), + undefined, + `${command} iOS sim (supported) hint`, + ); + } + // Commands without a hint hook return undefined (admission keeps its generic message). + assert.equal(unsupportedHintForDevice('tap', macOsDevice), undefined); +}); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index c67ca46e9..01e653431 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -12,6 +12,8 @@ export type CommandCapability = { android?: KindMatrix; linux?: KindMatrix; supports?: (device: DeviceInfo) => boolean; + /** Optional actionable hint surfaced when this command is rejected at admission for `device`. */ + unsupportedHint?: (device: DeviceInfo) => string | undefined; }; const isNotMacOs = (device: DeviceInfo): boolean => device.platform !== 'macos'; @@ -20,6 +22,19 @@ const isMacOsOrAppleSimulator = (device: DeviceInfo): boolean => const isIosMobileSimulator = (device: DeviceInfo): boolean => device.platform === 'ios' && device.kind === 'simulator' && device.target !== 'tv'; +// Two-finger gesture synthesis (RunnerSynthesizedGesture) is iOS-simulator-only (plus Android). +// When such a gesture is rejected at admission, explain where it IS available so an agent can +// redirect instead of getting a bare "not supported on this device". +const synthesisGestureUnsupportedHint = (device: DeviceInfo): string | undefined => { + if (device.platform === 'macos') + return 'macOS automation has no multi-touch input — this gesture is supported on Android and the iOS simulator only.'; + if (device.platform === 'ios' && device.target === 'tv') + return 'tvOS has no touch input — this gesture is supported on Android and the iOS simulator only.'; + if (device.platform === 'ios' && device.kind === 'device') + return 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.'; + return undefined; +}; + // Linux desktop supports these commands via xdotool/ydotool + AT-SPI2. // Linux device kind is always 'device' (local desktop). const LINUX_DEVICE: KindMatrix = { device: true }; @@ -61,18 +76,21 @@ const COMMAND_CAPABILITY_MATRIX: Record = { // it is excluded and fails fast at admission rather than round-tripping to an unsupported // runner. Matches rotate-gesture / transform-gesture. supports: (device) => device.platform === 'android' || isIosMobileSimulator(device), + unsupportedHint: synthesisGestureUnsupportedHint, }, 'rotate-gesture': { apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, supports: (device) => device.platform === 'android' || isIosMobileSimulator(device), + unsupportedHint: synthesisGestureUnsupportedHint, }, 'transform-gesture': { apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, supports: (device) => device.platform === 'android' || isIosMobileSimulator(device), + unsupportedHint: synthesisGestureUnsupportedHint, }, 'app-switcher': { apple: { simulator: true, device: true }, @@ -240,6 +258,10 @@ export function isCommandSupportedOnDevice(command: string, device: DeviceInfo): return byPlatform[kind] === true; } +export function unsupportedHintForDevice(command: string, device: DeviceInfo): string | undefined { + return COMMAND_CAPABILITY_MATRIX[command]?.unsupportedHint?.(device); +} + export function listCapabilityCommands(): string[] { return Object.keys(COMMAND_CAPABILITY_MATRIX).sort(); } diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index 54e50a73c..aed090d94 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -1,6 +1,6 @@ import { dispatchCommand, type CommandFlags } from '../core/dispatch.ts'; import { DAEMON_COMMAND_GROUPS, GESTURE_SUBCOMMAND_ERROR } from '../command-catalog.ts'; -import { isCommandSupportedOnDevice } from '../core/capabilities.ts'; +import { isCommandSupportedOnDevice, unsupportedHintForDevice } from '../core/capabilities.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonCommandContext } from './context.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; @@ -154,11 +154,13 @@ async function ensureGenericCommandReady( platformCommand: string, ): Promise { if (!isCommandSupportedOnDevice(platformCommand, session.device)) { + const hint = unsupportedHintForDevice(platformCommand, session.device); return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: `${platformCommand} is not supported on this device`, + ...(hint ? { hint } : {}), }, }; }