perf: tighten Maestro iOS stabilization

Author: thymikee

.github/workflows/ios.yml

@@ -71,7 +71,8 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSystemModalProbeSliceSharesAndClampsToPlanDeadline \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \
-            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds
 
       - name: Prepare iOS runner
         run: |

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -196,7 +196,12 @@ extension RunnerTests {
         regularSnapshot: true
       )
 
-      let key = "\(snapshot.elementType)-\(evaluation.label)-\(evaluation.identifier)-\(snapshot.frame.origin.x)-\(snapshot.frame.origin.y)"
+      let key = Self.snapshotTraversalIdentity(
+        elementType: snapshot.elementType,
+        label: evaluation.label,
+        identifier: evaluation.identifier,
+        frame: snapshot.frame
+      )
       let isDuplicate = seen.contains(key)
       if !isDuplicate {
         seen.insert(key)
@@ -1118,6 +1123,15 @@ extension RunnerTests {
     return .infinite
   }
 
+  static func snapshotTraversalIdentity(
+    elementType: XCUIElement.ElementType,
+    label: String,
+    identifier: String,
+    frame: CGRect
+  ) -> String {
+    "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)-\(frame.width)-\(frame.height)"
+  }
+
   private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
     if depth > 4 { return nil }
     let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)

apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift

@@ -0,0 +1,20 @@
+import XCTest
+
+extension RunnerTests {
+  func testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds() {
+    let wrapper = Self.snapshotTraversalIdentity(
+      elementType: .other,
+      label: "Article",
+      identifier: "",
+      frame: CGRect(x: 0, y: 97, width: 393, height: 48)
+    )
+    let leaf = Self.snapshotTraversalIdentity(
+      elementType: .other,
+      label: "Article",
+      identifier: "",
+      frame: CGRect(x: 0, y: 97, width: 131, height: 48)
+    )
+
+    XCTAssertNotEqual(wrapper, leaf)
+  }
+}

docs/adr/0015-direct-maestro-engine.md

@@ -66,10 +66,12 @@ are failure/debug artifacts, not happy-path requirements.
 
 The daemon adapter may retain the provider snapshot behind a successful observation without exposing it
 through the engine contract. A following target resolution may use that snapshot only as semantic
-evidence. Maestro selector semantics continue to use the raw provider nodes, except for equivalent iOS
-accessibility wrapper chains described below. On iOS, the adapter derives the provider's canonical
-interactive presentation from that same snapshot solely to determine atomic selector uniqueness; it
-does not perform another capture. A unique exact, canonically hittable match may be dispatched with its
+evidence. Maestro selector semantics use the shared structurally normalized provider snapshot, before
+iOS canonical interactive presentation. They deliberately do not request the public `snapshot --raw`
+shape, whose output contract bypasses shared pruning and overlay annotations. On iOS, the adapter derives the provider's canonical
+interactive presentation from that same snapshot without another capture. Raw matching still selects
+the node and authored index; when that same source node has canonical interactive bounds, tap-like
+actions use those bounds. A unique exact, canonically hittable match may be dispatched with its
 resolved point so XCTest binds the live selector identity and coordinate delivery atomically; a
 structured live-selector miss, ambiguity, point mismatch, or off-screen result
 falls back to fresh Maestro resolution. All other targets capture fresh geometry before coordinate
@@ -87,22 +89,33 @@ matches. It does not adopt the public agent-device command surface's unique-matc
 suppressed by Maestro `optional`; atomic iOS dispatch handles that result by performing fresh Maestro
 resolution, as described above. Cancellation and infrastructure failures are likewise non-optional.
 
-Every `tapOn` waits for hierarchy stability before the next command, matching Maestro's tap boundary.
-`retryTapIfNoChange` defaults to false; when explicitly enabled, the runtime compares the stable hierarchy
-with the target-resolution hierarchy and retries once when nothing changed. The stable result primes the
-next command, so the tap boundary does not add another hierarchy read there.
-
-Successful gestures and scrolls require stabilization before the next command executes. The runtime
-port records that requirement without capturing a hierarchy in the gesture command itself. At the next
-command boundary, it samples at the compatibility polling cadence until the observation is stable, then
-retains that final observation for the immediately following read. This keeps mutation ordering explicit,
-prevents a later gesture or assertion from overtaking an in-flight transition, and avoids paying for the
-same stable observation twice.
+Mutations that can leave an in-flight transition record a pending stabilization boundary. Before another
+mutating command, the runtime samples at the compatibility polling cadence until the hierarchy is stable
+and retains that final observation for the next read. Commands with specialized completion semantics settle
+inline instead. Hierarchy signatures project the available semantic attributes, tree topology, and
+fixed-order integer edges while excluding provider-only metadata, object-key order, and subpixel noise.
+
+An authored observation after a mutation skips the pending barrier and polls its own condition immediately.
+Its successful snapshot becomes the baseline for the boundary but does not discharge mutation ordering. If
+another mutation follows, the runtime waits one polling interval and compares a fresh hierarchy with that
+baseline, continuing until stable when the UI is still changing. If no later mutation follows, it performs no
+extra capture. This intentionally differs from upstream Maestro, which settles before evaluating an assertion,
+and retains the previous agent-device compatibility engine's faster read behavior without allowing a later
+tap, gesture, scroll, or input to overtake an in-flight transition.
+
+`retryTapIfNoChange` defaults to false. When explicitly enabled, the runtime compares the stable hierarchy
+with the target-resolution hierarchy. On iOS, an unchanged hierarchy triggers the same screenshot comparison
+used by Maestro; the runtime retries once only when both surfaces remain unchanged. Screenshot evidence is
+best-effort because a failed capture cannot prove that repeating a mutation is safe. The stable hierarchy
+primes the next command, so the retry policy does not add another hierarchy read there.
 
 `waitForAnimationToEnd` uses its own screenshot-stability operation, matching upstream's two immediate
 captures per attempt and 0.005% normalized absolute RGB-difference threshold. These captures explicitly
 bypass ordinary screenshot stabilization so the command observes the application rather than
-recursively waiting on another settling policy.
+recursively waiting on another settling policy. Screenshot stability does not discharge a pending
+mutation boundary; a later mutating command still verifies hierarchy stability before dispatch. On iOS,
+comparison captures use the persistent runner's screenshot surface so both frames come from one warmed
+transport and avoid simulator screenshot setup between polls.
 
 Upstream Maestro is a version-pinned development reference, not a production dependency. The current
 opt-in conformance script compares a small checked-in parser corpus against manually captured Maestro
@@ -118,14 +131,15 @@ The migration cannot switch production routing until Android and iOS satisfy all
 and react-navigation corpora:
 
 - total wall time is no slower than the pre-migration compatibility engine;
-- target interactions perform one outcome query after the compatibility polling interval and prime that
-  result for the next command;
+- an observation immediately after a mutation polls its authored condition directly; a later mutation uses
+  that result as its stabilization baseline;
 - no command captures a second hierarchy merely to re-verify evidence produced within that command;
 - absolute coordinate swipes perform one direct viewport query and no accessibility capture;
-- gesture stabilization is deferred to the next command boundary and primes the following observation;
+- mutation-to-mutation stabilization is deferred to the next command boundary and primes the following read;
 - percentage swipe conversion preserves authored endpoints exactly;
 - helper/runner startup remains amortized across a suite;
-- p50/p95 command latency, captures, retries, and transferred hierarchy bytes are reported separately;
+- p50/p95 command latency, hierarchy and screenshot captures, tap retries, and transferred hierarchy
+  bytes are reported separately;
 - failure-only diagnostics are excluded from happy-path latency comparisons.
 
 Android verification must prove the bundled helper backend and version. iOS verification must separate

src/compat/maestro/__tests__/daemon-runtime-port-mutations.test.ts

@@ -0,0 +1,203 @@
+import { expect, test } from 'vitest';
+import type { DaemonRequest } from '../../../daemon/types.ts';
+import { createDaemonMaestroRuntimePort } from '../daemon-runtime-port.ts';
+import { MAESTRO_OBSERVATION_POLL_MS } from '../daemon-runtime-port-observation.ts';
+import type { MaestroRuntimeCommand } from '../engine-types.ts';
+import { makeBaseRequest, makeDependencies } from './daemon-runtime-port-fixtures.ts';
+
+test.each([
+  {
+    name: 'double tap',
+    command: {
+      kind: 'doubleTapOn',
+      source: { line: 2 },
+      target: { space: 'absolute', x: 100, y: 200 },
+    },
+  },
+  {
+    name: 'long press',
+    command: {
+      kind: 'longPressOn',
+      source: { line: 2 },
+      target: { space: 'absolute', x: 100, y: 200 },
+    },
+  },
+] satisfies { name: string; command: MaestroRuntimeCommand }[])(
+  'settles $name before a following gesture',
+  async ({ command }) => {
+    const requests: DaemonRequest[] = [];
+    const clock = { value: 0 };
+    const port = createDaemonMaestroRuntimePort({
+      baseReq: makeBaseRequest({ flags: { platform: 'ios', replayBackend: 'maestro' } }),
+      invoke: async (request) => {
+        requests.push(request);
+        return request.command === 'snapshot'
+          ? {
+              ok: true,
+              data: {
+                nodes: [
+                  {
+                    index: 0,
+                    type: 'Application',
+                    rect: { x: 0, y: 0, width: 393, height: 852 },
+                  },
+                ],
+              },
+            }
+          : { ok: true, data: {} };
+      },
+      dependencies: makeDependencies(clock),
+      platform: 'ios',
+    });
+
+    await port.execute({
+      command,
+      generation: 0,
+      env: {},
+      invalidateObservation() {},
+    });
+    await port.execute({
+      command: {
+        kind: 'swipe',
+        source: { line: 3 },
+        gesture: {
+          kind: 'coordinates',
+          start: { space: 'absolute', x: 350, y: 400 },
+          end: { space: 'absolute', x: 40, y: 400 },
+          duration: 100,
+        },
+      },
+      generation: 1,
+      env: {},
+      invalidateObservation() {},
+    });
+
+    expect(requests.map(({ command: requestCommand }) => requestCommand)).toEqual([
+      'click',
+      'snapshot',
+      'snapshot',
+      'swipe',
+    ]);
+    expect(clock.value).toBe(MAESTRO_OBSERVATION_POLL_MS);
+  },
+);
+
+test('preserves mutation ordering after a failed dispatch is continued', async () => {
+  const requests: DaemonRequest[] = [];
+  const clock = { value: 0 };
+  let swipeAttempts = 0;
+  const port = createDaemonMaestroRuntimePort({
+    baseReq: makeBaseRequest({ flags: { platform: 'ios', replayBackend: 'maestro' } }),
+    invoke: async (request) => {
+      requests.push(request);
+      if (request.command === 'swipe' && swipeAttempts++ === 0) {
+        throw new Error('dispatch failed after delivery became possible');
+      }
+      return request.command === 'snapshot'
+        ? {
+            ok: true,
+            data: {
+              nodes: [
+                {
+                  index: 0,
+                  type: 'Application',
+                  rect: { x: 0, y: 0, width: 393, height: 852 },
+                },
+              ],
+            },
+          }
+        : { ok: true, data: {} };
+    },
+    dependencies: makeDependencies(clock),
+    platform: 'ios',
+  });
+  const swipe = (generation: number) =>
+    port.execute({
+      command: {
+        kind: 'swipe',
+        source: { line: generation + 2 },
+        gesture: {
+          kind: 'coordinates',
+          start: { space: 'absolute', x: 350, y: 400 },
+          end: { space: 'absolute', x: 40, y: 400 },
+          duration: 100,
+        },
+      },
+      generation,
+      env: {},
+      invalidateObservation() {},
+    });
+
+  await expect(swipe(0)).rejects.toThrow('dispatch failed');
+  await swipe(1);
+
+  expect(requests.map(({ command }) => command)).toEqual([
+    'swipe',
+    'snapshot',
+    'snapshot',
+    'swipe',
+  ]);
+  expect(clock.value).toBe(MAESTRO_OBSERVATION_POLL_MS);
+});
+
+test('defers a pending mutation boundary across non-mutating runtime commands', async () => {
+  const requests: DaemonRequest[] = [];
+  const clock = { value: 0 };
+  const port = createDaemonMaestroRuntimePort({
+    baseReq: makeBaseRequest({ flags: { platform: 'ios', replayBackend: 'maestro' } }),
+    invoke: async (request) => {
+      requests.push(request);
+      return request.command === 'snapshot'
+        ? {
+            ok: true,
+            data: {
+              nodes: [
+                {
+                  index: 0,
+                  type: 'Application',
+                  rect: { x: 0, y: 0, width: 393, height: 852 },
+                },
+              ],
+            },
+          }
+        : { ok: true, data: {} };
+    },
+    dependencies: makeDependencies(clock),
+    platform: 'ios',
+  });
+  const swipe = (generation: number) =>
+    port.execute({
+      command: {
+        kind: 'swipe',
+        source: { line: generation + 2 },
+        gesture: {
+          kind: 'coordinates',
+          start: { space: 'absolute', x: 350, y: 400 },
+          end: { space: 'absolute', x: 40, y: 400 },
+          duration: 100,
+        },
+      },
+      generation,
+      env: {},
+      invalidateObservation() {},
+    });
+
+  await swipe(0);
+  await port.execute({
+    command: { kind: 'takeScreenshot', source: { line: 3 }, path: 'state.png' },
+    generation: 1,
+    env: {},
+    invalidateObservation() {},
+  });
+  expect(requests.map(({ command }) => command)).toEqual(['swipe', 'screenshot']);
+
+  await swipe(1);
+  expect(requests.map(({ command }) => command)).toEqual([
+    'swipe',
+    'screenshot',
+    'snapshot',
+    'snapshot',
+    'swipe',
+  ]);
+  expect(clock.value).toBe(MAESTRO_OBSERVATION_POLL_MS);
+});