fix: stabilize iOS runner gestures (#800)

* fix: avoid stale iOS window viewport queries

* fix: synthesize iOS swipe gestures

* fix: honor iOS drag gesture durations

* test: update iOS gesture provider transcript

* fix: align sequence drag fallback timing

Author: thymikee

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift

@@ -49,10 +49,9 @@ extension RunnerTests {
         return nil
       }
 
-      // The public windows query backing safeSnapshotViewport can fail on the same apps that
-      // need this fallback, degrading to an infinite viewport that marks off-screen content
-      // (e.g. closed drawer menus at negative x) as visible and tappable. The private root's
-      // own frame is the reliable screen rect here.
+      // If the app frame is unavailable, the private root's own frame is the reliable screen
+      // rect here. Avoid public window queries: stale transient windows can record XCTest
+      // failures after the runner already returned a successful command response.
       var viewport = safeSnapshotViewport(app: app)
       let rootFrame = privateAXRect(root["frame"])
       if viewport.isInfinite || viewport.isNull || viewport.isEmpty, !rootFrame.isEmpty {

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -13,7 +13,7 @@ extension RunnerTests {
     return (gestureStartUptimeMs, currentUptimeMs())
   }
 
-  private func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
+  func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
     min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120)
   }
 

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -157,8 +157,8 @@ struct SequenceStep: Codable {
   let y2: Double?
   let durationMs: Double?
   let pauseMs: Double?
-  /// For `tap` steps on iOS non-tv: use the synthesized HID tap fast path (synthesizedTapAt)
-  /// instead of the drag-based XCUICoordinate tapAt, matching the individual `tap` command.
+  /// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the
+  /// drag-based XCUICoordinate path, matching the individual command behavior.
   let synthesized: Bool?
 }
 

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift

@@ -164,6 +164,50 @@ extension RunnerTests {
       }
       // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below.
     }
+    if step.kind == "drag", step.synthesized == true {
+      let dragPoints = keyboardAvoidingDragPoints(
+        app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y)
+      let durationMs = min(max(step.durationMs ?? 250, 16), 10000)
+      let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+        synthesizedDragAt(
+          app: activeApp,
+          x: dragPoints.x,
+          y: dragPoints.y,
+          x2: dragPoints.x2,
+          y2: dragPoints.y2,
+          durationMs: durationMs
+        )
+      }
+      if case .performed = outcome {
+        if let pauseMs = step.pauseMs, pauseMs > 0 {
+          sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
+        }
+        return SequenceStepOutcome(
+          outcome: outcome,
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      }
+      let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250)
+      let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) {
+        dragAt(
+          app: activeApp,
+          x: dragPoints.x,
+          y: dragPoints.y,
+          x2: dragPoints.x2,
+          y2: dragPoints.y2,
+          holdDuration: fallbackHoldDuration
+        )
+      }
+      if case .performed = fallbackOutcome, let pauseMs = step.pauseMs, pauseMs > 0 {
+        sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
+      }
+      return SequenceStepOutcome(
+        outcome: fallbackOutcome,
+        gestureStartUptimeMs: fallbackTiming.gestureStartUptimeMs,
+        gestureEndUptimeMs: fallbackTiming.gestureEndUptimeMs
+      )
+    }
     let (timing, outcome) = performGesture(activeApp) {
       switch step.kind {
       case "doubleTap":
@@ -173,9 +217,9 @@ extension RunnerTests {
         let duration = min(max(step.durationMs ?? 800, 16), 10000) / 1000.0
         return longPressAt(app: activeApp, x: x, y: y, duration: duration)
       case "drag":
-        // Route through keyboardAvoidingDragPoints for parity with the individual `.drag` command
-        // (RunnerTests+CommandExecution.swift). durationMs is intentionally ignored on this
-        // coordinate-drag path, matching that command's non-synthesized branch.
+        // Route through keyboardAvoidingDragPoints for parity with the individual `.drag` command.
+        // The non-synthesized coordinate-drag path ignores durationMs, matching that command's
+        // non-synthesized branch.
         let dragPoints = keyboardAvoidingDragPoints(
           app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y)
         return dragAt(

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -739,15 +739,6 @@ extension RunnerTests {
   }
 
   private func snapshotViewport(app: XCUIApplication) -> CGRect {
-    let windows = app.windows.allElementsBoundByIndex
-    let windowFrames = windows
-      .filter { $0.exists && !$0.frame.isNull && !$0.frame.isEmpty }
-      .map(\.frame)
-    if let largestWindowFrame = windowFrames.max(by: { left, right in
-      left.width * left.height < right.width * right.height
-    }) {
-      return largestWindowFrame
-    }
     let appFrame = app.frame
     if !appFrame.isNull && !appFrame.isEmpty {
       return appFrame

src/core/__tests__/dispatch-interactions.test.ts

@@ -197,8 +197,25 @@ test('handleSwipeCommand fuses repeated swipes into sequence drag steps with pin
     steps: [
       // Ping-pong is unrolled daemon-side: odd indices swap endpoints, replacing the
       // runner-side pattern handling of the retired dragSeries command.
-      { kind: 'drag', x: 100, y: 650, x2: 100, y2: 450, durationMs: 120, pauseMs: 50 },
-      { kind: 'drag', x: 100, y: 450, x2: 100, y2: 650, durationMs: 120 },
+      {
+        kind: 'drag',
+        x: 100,
+        y: 650,
+        x2: 100,
+        y2: 450,
+        durationMs: 120,
+        synthesized: true,
+        pauseMs: 50,
+      },
+      {
+        kind: 'drag',
+        x: 100,
+        y: 450,
+        x2: 100,
+        y2: 650,
+        durationMs: 120,
+        synthesized: true,
+      },
     ],
     appBundleId: 'com.example.App',
   });

src/core/dispatch-interactions.ts

@@ -427,10 +427,10 @@ function buildPressSequenceSteps(
 
 // Unrolls a swipe series into `sequence` drag steps, replacing the retired `dragSeries` runner
 // command. Ping-pong becomes per-step endpoint swapping (odd indices reversed), matching the
-// runner-side performDragSeries the daemon no longer invokes. durationMs is carried for wire
-// fidelity and budget estimation; the runner's coordinate-drag path ignores it, exactly as the
-// daemon-sent (non-synthesized) dragSeries did.
+// runner-side performDragSeries the daemon no longer invokes. iOS touch targets request the same
+// synthesized, duration-aware drag path as one-shot swipe; macOS/tvOS keep coordinate drag.
 function buildSwipeSequenceSteps(params: {
+  device: DeviceInfo;
   x1: number;
   y1: number;
   x2: number;
@@ -440,7 +440,8 @@ function buildSwipeSequenceSteps(params: {
   pattern: string;
   effectiveDurationMs: number;
 }): RunnerSequenceStep[] {
-  const { x1, y1, x2, y2, count, pauseMs, pattern, effectiveDurationMs } = params;
+  const { device, x1, y1, x2, y2, count, pauseMs, pattern, effectiveDurationMs } = params;
+  const synthesized = device.platform === 'ios' && device.target !== 'tv';
   return Array.from({ length: count }, (_, index) => {
     const reverse = pattern === 'ping-pong' && index % 2 === 1;
     const isLast = index === count - 1;
@@ -451,6 +452,7 @@ function buildSwipeSequenceSteps(params: {
       x2: reverse ? x1 : x2,
       y2: reverse ? y1 : y2,
       durationMs: effectiveDurationMs,
+      ...(synthesized ? { synthesized: true } : {}),
       ...(!isLast && pauseMs > 0 ? { pauseMs } : {}),
     };
   });
@@ -608,7 +610,17 @@ async function runSwipeCoordinates(params: {
   if (shouldUseIosDragSeries(device, count)) {
     const aggregated = await runIosSequenceChunks(
       device,
-      buildSwipeSequenceSteps({ x1, y1, x2, y2, count, pauseMs, pattern, effectiveDurationMs }),
+      buildSwipeSequenceSteps({
+        device,
+        x1,
+        y1,
+        x2,
+        y2,
+        count,
+        pauseMs,
+        pattern,
+        effectiveDurationMs,
+      }),
       context,
     );
     return {

src/platforms/ios/__tests__/index.test.ts

@@ -184,7 +184,7 @@ test('resolveMacOsHelperPackageRootFrom finds helper package from source and dis
   }
 });
 
-test('iosRunnerOverrides gives fling a short default XCUITest drag hold', async () => {
+test('iosRunnerOverrides maps iOS fling duration to synthesized drag', async () => {
   mockRunIosRunnerCommand.mockResolvedValue({});
 
   const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, {
@@ -200,6 +200,7 @@ test('iosRunnerOverrides gives fling a short default XCUITest drag hold', async
     x2: 180,
     y2: 200,
     durationMs: 16,
+    synthesized: true,
     appBundleId: 'com.example.App',
   });
 });
@@ -252,7 +253,7 @@ for (const [name, device] of [
   });
 }
 
-test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => {
+test('iosRunnerOverrides maps iOS swipe and pan durations to synthesized drag', async () => {
   mockRunIosRunnerCommand.mockResolvedValue({});
 
   const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, {
@@ -270,6 +271,7 @@ test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => {
     x2: 180,
     y2: 200,
     durationMs: 300,
+    synthesized: true,
     appBundleId: 'com.example.App',
   });
   assert.deepEqual(mockRunIosRunnerCommand.mock.calls[1]?.[1], {
@@ -279,6 +281,7 @@ test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => {
     x2: 180,
     y2: 200,
     durationMs: 250,
+    synthesized: true,
     appBundleId: 'com.example.App',
   });
   assert.deepEqual(mockRunIosRunnerCommand.mock.calls[2]?.[1], {
@@ -288,6 +291,7 @@ test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => {
     x2: 180,
     y2: 200,
     durationMs: 300,
+    synthesized: true,
     appBundleId: 'com.example.App',
   });
 });
@@ -296,14 +300,16 @@ for (const [name, device] of [
   ['macOS', MACOS_TEST_DEVICE],
   ['tvOS', TVOS_TEST_SIMULATOR],
 ] as const) {
-  test(`iosRunnerOverrides keeps ${name} swipes on the standard drag path`, async () => {
+  test(`iosRunnerOverrides keeps ${name} drag gestures on the standard path`, async () => {
     mockRunIosRunnerCommand.mockResolvedValue({});
 
     const { overrides } = iosRunnerOverrides(device, {
       appBundleId: 'com.example.App',
     });
 
     await overrides.swipe(100, 200, 180, 200, 300);
+    await overrides.pan(100, 200, 180, 200, 300);
+    await overrides.fling(100, 200, 180, 200, 300);
 
     assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], {
       command: 'drag',
@@ -314,6 +320,24 @@ for (const [name, device] of [
       durationMs: 300,
       appBundleId: 'com.example.App',
     });
+    assert.deepEqual(mockRunIosRunnerCommand.mock.calls[1]?.[1], {
+      command: 'drag',
+      x: 100,
+      y: 200,
+      x2: 180,
+      y2: 200,
+      durationMs: 300,
+      appBundleId: 'com.example.App',
+    });
+    assert.deepEqual(mockRunIosRunnerCommand.mock.calls[2]?.[1], {
+      command: 'drag',
+      x: 100,
+      y: 200,
+      x2: 180,
+      y2: 200,
+      durationMs: 300,
+      appBundleId: 'com.example.App',
+    });
   });
 }
 

src/platforms/ios/interactions.ts

@@ -29,6 +29,12 @@ type NormalizedScrollOptions = {
   preferProvidedPixels?: boolean;
 };
 
+type IosDragCommandOptions = {
+  defaultDurationMs: number;
+  legacyDefaultDurationMs?: number;
+  synthesized?: boolean;
+};
+
 type IosRunnerOverrides = Pick<
   Interactor,
   | 'tap'
@@ -101,6 +107,7 @@ export function iosRunnerOverrides(
           device,
           iosDragCommand(device, ctx, x1, y1, x2, y2, durationMs, {
             defaultDurationMs: IOS_SWIPE_DEFAULT_DURATION_MS,
+            synthesized: shouldUseSynthesizedIosGesture(device),
           }),
           runnerOpts,
         );
@@ -111,22 +118,19 @@ export function iosRunnerOverrides(
           iosDragCommand(device, ctx, x1, y1, x2, y2, durationMs, {
             defaultDurationMs: 500,
             legacyDefaultDurationMs: 500,
+            synthesized: shouldUseSynthesizedIosGesture(device),
           }),
           runnerOpts,
         );
       },
       fling: async (x1, y1, x2, y2, durationMs) => {
         return await runIosRunnerCommand(
           device,
-          {
-            command: 'drag',
-            x: x1,
-            y: y1,
-            x2,
-            y2,
-            durationMs: durationMs ?? 16,
-            appBundleId: ctx.appBundleId,
-          },
+          iosDragCommand(device, ctx, x1, y1, x2, y2, durationMs, {
+            defaultDurationMs: 16,
+            legacyDefaultDurationMs: 16,
+            synthesized: shouldUseSynthesizedIosGesture(device),
+          }),
           runnerOpts,
         );
       },
@@ -252,11 +256,15 @@ function iosTapCommand(
     command: 'tap',
     x,
     y,
-    ...(device.platform === 'ios' && device.target !== 'tv' ? { synthesized: true } : {}),
+    ...(shouldUseSynthesizedIosGesture(device) ? { synthesized: true } : {}),
     appBundleId: ctx.appBundleId,
   };
 }
 
+function shouldUseSynthesizedIosGesture(device: DeviceInfo): boolean {
+  return device.platform === 'ios' && device.target !== 'tv';
+}
+
 function iosDragCommand(
   device: DeviceInfo,
   ctx: RunnerContext,
@@ -265,10 +273,7 @@ function iosDragCommand(
   x2: number,
   y2: number,
   durationMs: number | undefined,
-  options: {
-    defaultDurationMs: number;
-    legacyDefaultDurationMs?: number;
-  },
+  options: IosDragCommandOptions,
 ): RunnerCommand {
   const normalizedDurationMs =
     device.platform === 'ios' && device.target !== 'tv'
@@ -281,6 +286,7 @@ function iosDragCommand(
     x2,
     y2,
     ...(normalizedDurationMs !== undefined ? { durationMs: normalizedDurationMs } : {}),
+    ...(options.synthesized === true ? { synthesized: true } : {}),
     appBundleId: ctx.appBundleId,
   };
 }

test/integration/provider-scenarios/ios-world.ts

@@ -76,6 +76,7 @@ export async function createIosSettingsWorld(): Promise<IosSettingsWorld> {
         x2: 276,
         y2: 122,
         durationMs: 500,
+        synthesized: true,
         appBundleId: 'com.apple.Preferences',
       },
       result: { dragged: true },
@@ -91,6 +92,7 @@ export async function createIosSettingsWorld(): Promise<IosSettingsWorld> {
         x2: 376,
         y2: 122,
         durationMs: 50,
+        synthesized: true,
         appBundleId: 'com.apple.Preferences',
       },
       result: { flung: true },