fix: retry no-op maestro taps

Author: thymikee

src/compat/maestro/runtime-interactions.ts

@@ -502,6 +502,10 @@ async function clickMaestroSnapshotTarget(
     ...params.baseReq,
     command: 'click',
     positionals: [String(point.x), String(point.y)],
+    flags: {
+      ...params.baseReq.flags,
+      interactionOutcome: { retryOnNoChange: true },
+    },
   });
   if (response.ok) clearMaestroVisibleContext(params.scope);
   return {
@@ -550,6 +554,7 @@ async function invokeMaestroFuzzyTapOn(
     flags: {
       ...params.baseReq.flags,
       findFirst: true,
+      interactionOutcome: { retryOnNoChange: true },
     },
   });
   if (findResponse.ok) return { retry: false, response: findResponse };

src/core/dispatch-context.ts

@@ -15,6 +15,9 @@ export type MaestroRuntimeFlags = {
 export type CommandFlags = Omit<CliFlags, DaemonExcludedCliFlag> & {
   batchSteps?: DaemonBatchStep[];
   clearAppState?: boolean;
+  interactionOutcome?: {
+    retryOnNoChange?: boolean;
+  };
   launchArgs?: string[];
   maestro?: MaestroRuntimeFlags;
   replayBackend?: string;

src/daemon/__tests__/interaction-outcome-policy.test.ts

@@ -0,0 +1,128 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import type { SnapshotState } from '../../utils/snapshot.ts';
+import {
+  buildInteractionSurfaceSignature,
+  classifyInteractionSurfaceChange,
+  markPendingInteractionOutcome,
+  stripInternalInteractionOutcomeFlags,
+} from '../interaction-outcome-policy.ts';
+import type { SessionState } from '../types.ts';
+import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
+
+test('classifyInteractionSurfaceChange treats identical surfaces as unchanged', () => {
+  const before = buildInteractionSurfaceSignature(makeSnapshot('Inbox').nodes);
+  const after = buildInteractionSurfaceSignature(makeSnapshot('Inbox').nodes);
+
+  assert.equal(classifyInteractionSurfaceChange(before, after), 'unchanged');
+});
+
+test('classifyInteractionSurfaceChange tolerates tiny rect drift', () => {
+  const before = buildInteractionSurfaceSignature(makeSnapshot('Inbox', 100).nodes);
+  const after = buildInteractionSurfaceSignature(makeSnapshot('Inbox', 100.4).nodes);
+
+  assert.equal(classifyInteractionSurfaceChange(before, after), 'unchanged');
+});
+
+test('classifyInteractionSurfaceChange detects semantic screen changes', () => {
+  const before = buildInteractionSurfaceSignature(makeSnapshot('Inbox').nodes);
+  const after = buildInteractionSurfaceSignature(makeSnapshot('Article detail').nodes);
+
+  assert.equal(classifyInteractionSurfaceChange(before, after), 'changed');
+});
+
+test('classifyInteractionSurfaceChange detects material layout movement', () => {
+  const before = buildInteractionSurfaceSignature(makeSnapshot('Inbox', 100).nodes);
+  const after = buildInteractionSurfaceSignature(makeSnapshot('Inbox', 180).nodes);
+
+  assert.equal(classifyInteractionSurfaceChange(before, after), 'changed');
+});
+
+test('markPendingInteractionOutcome stores retry state only for explicit retry flags', () => {
+  const session = makeSession();
+  markPendingInteractionOutcome({
+    session,
+    command: 'click',
+    positionals: ['20', '40'],
+    flags: {},
+    preSnapshot: makeSnapshot('Inbox'),
+  });
+  assert.equal(session.pendingInteractionOutcome, undefined);
+
+  const retrySession = makeSession();
+  markPendingInteractionOutcome({
+    session: retrySession,
+    command: 'click',
+    positionals: ['20', '40'],
+    flags: { interactionOutcome: { retryOnNoChange: true } },
+    preSnapshot: makeSnapshot('Inbox'),
+  });
+
+  assert.equal(retrySession.pendingInteractionOutcome?.command, 'click');
+  assert.equal(retrySession.pendingInteractionOutcome?.attemptsRemaining, 2);
+  assert.equal(retrySession.pendingInteractionOutcome?.flags?.interactionOutcome, undefined);
+
+  const refSession = makeSession();
+  markPendingInteractionOutcome({
+    session: refSession,
+    command: 'click',
+    positionals: ['@e1'],
+    flags: { interactionOutcome: { retryOnNoChange: true } },
+    preSnapshot: makeSnapshot('Inbox'),
+  });
+  assert.equal(refSession.pendingInteractionOutcome, undefined);
+
+  const longPressSession = makeSession();
+  markPendingInteractionOutcome({
+    session: longPressSession,
+    command: 'longpress',
+    positionals: ['20', '40', '800'],
+    flags: { interactionOutcome: { retryOnNoChange: true } },
+    preSnapshot: makeSnapshot('Inbox'),
+  });
+  assert.equal(longPressSession.pendingInteractionOutcome, undefined);
+});
+
+test('stripInternalInteractionOutcomeFlags removes internal retry controls', () => {
+  assert.deepEqual(
+    stripInternalInteractionOutcomeFlags({
+      platform: 'ios',
+      interactionOutcome: { retryOnNoChange: true },
+    }),
+    { platform: 'ios' },
+  );
+});
+
+function makeSession(): SessionState {
+  return {
+    name: 'ios',
+    device: IOS_SIMULATOR,
+    createdAt: Date.now(),
+    actions: [],
+  };
+}
+
+function makeSnapshot(label: string, y = 100): SnapshotState {
+  return {
+    nodes: [
+      {
+        ref: 'e1',
+        index: 0,
+        type: 'Application',
+        label: 'App',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+      },
+      {
+        ref: 'e2',
+        index: 1,
+        parentIndex: 0,
+        type: 'Button',
+        identifier: 'primary-action',
+        label,
+        rect: { x: 120, y, width: 80, height: 40 },
+      },
+    ],
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+}

src/daemon/handlers/__tests__/find.test.ts

@@ -36,10 +36,12 @@ async function runFindClickScenario(options: {
 }): Promise<{
   response: NonNullable<Awaited<ReturnType<typeof handleFindCommands>>>;
   invokeCalls: DaemonRequest[];
+  session: SessionState;
 }> {
   const sessionStore = makeSessionStore();
   const sessionName = 'default';
-  sessionStore.set(sessionName, options.session ?? makeSession(sessionName));
+  const session = options.session ?? makeSession(sessionName);
+  sessionStore.set(sessionName, session);
 
   if (options.nodes !== undefined) {
     mockDispatch.mockImplementation(async (_device, command) => {
@@ -70,7 +72,7 @@ async function runFindClickScenario(options: {
   });
 
   expect(response).toBeTruthy();
-  return { response: response!, invokeCalls };
+  return { response: response!, invokeCalls, session };
 }
 
 test('handleFindCommands click returns deterministic metadata across locator variants', async () => {
@@ -213,6 +215,36 @@ test('handleFindCommands click prefers semantic controls over matching container
   expect(invokeCalls[0]!.positionals?.[0]).toBe('@e5');
 });
 
+test('handleFindCommands forwards internal interaction outcome flags only to delegated click', async () => {
+  const { response, invokeCalls, session } = await runFindClickScenario({
+    positionals: ['Continue', 'click'],
+    flags: {
+      findFirst: true,
+      interactionOutcome: { retryOnNoChange: true },
+    },
+    nodes: [
+      {
+        index: 0,
+        ref: 'e1',
+        type: 'Application',
+        rect: { x: 0, y: 0, width: 440, height: 956 },
+      },
+      {
+        index: 1,
+        ref: 'e2',
+        type: 'Button',
+        label: 'Continue',
+        rect: { x: 40, y: 870, width: 360, height: 44 },
+        parentIndex: 0,
+      },
+    ],
+  });
+
+  expect(response.ok).toBe(true);
+  expect(invokeCalls[0]!.flags?.interactionOutcome).toEqual({ retryOnNoChange: true });
+  expect(session.actions.at(-1)?.flags).toEqual({});
+});
+
 test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => {
   const sessionName = 'android-find-wait';
   const session: SessionState = {

src/daemon/handlers/__tests__/interaction.test.ts

@@ -1163,6 +1163,49 @@ test('press @ref preserves native timing in recorded result and touch visualizat
   expect(stored?.recording?.gestureEvents[0]?.tMs).toBe(570);
 });
 
+test('press @ref stores resolved coordinate retry payload for lazy outcome retry', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'retry-ref';
+  const session = makeSession(sessionName);
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'XCUIElementTypeButton',
+        label: 'Continue',
+        identifier: 'auth_continue',
+        rect: { x: 10, y: 20, width: 100, height: 40 },
+        enabled: true,
+        hittable: true,
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  sessionStore.set(sessionName, session);
+  mockDispatch.mockResolvedValue({});
+
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'press',
+      positionals: ['@e1'],
+      flags: { interactionOutcome: { retryOnNoChange: true } },
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+  });
+
+  expect(response?.ok).toBe(true);
+  const stored = sessionStore.get(sessionName);
+  expect(stored?.pendingInteractionOutcome?.command).toBe('press');
+  expect(stored?.pendingInteractionOutcome?.positionals).toEqual(['60', '40']);
+  expect(stored?.actions[0]?.positionals).toEqual(['@e1']);
+  expect(stored?.actions[0]?.flags).toEqual({});
+});
+
 test('longpress @ref resolves the target and dispatches coordinate longpress', async () => {
   const sessionStore = makeSessionStore();
   const sessionName = 'longpress-ref';

src/daemon/handlers/__tests__/session-replay-vars.test.ts

@@ -1886,6 +1886,12 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen
       ['find', ['Continue', 'click']],
     ],
   );
+  assert.deepEqual(calls.find((call) => call.command === 'click')?.flags?.interactionOutcome, {
+    retryOnNoChange: true,
+  });
+  assert.deepEqual(calls.find((call) => call.command === 'find')?.flags?.interactionOutcome, {
+    retryOnNoChange: true,
+  });
 });
 
 test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.when', async () => {

src/daemon/handlers/__tests__/snapshot-handler.test.ts

@@ -720,6 +720,87 @@ test('Android ref refresh mode does not retry narrow snapshots as sharp drops',
   expect(session.androidSnapshotFreshness).toBeUndefined();
 });
 
+test('captureSnapshot lazily retries pending no-change touch before returning fresh state', async () => {
+  const sessionName = 'ios-lazy-outcome-retry';
+  const session = makeSession(sessionName, iosSimulatorDevice);
+  const baselineNodes = [
+    {
+      ref: 'e1',
+      index: 0,
+      depth: 0,
+      type: 'Button',
+      label: 'Open feed',
+      identifier: 'open-feed',
+      hittable: true,
+      rect: { x: 20, y: 120, width: 160, height: 48 },
+    },
+  ];
+  session.snapshot = {
+    nodes: baselineNodes,
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  session.pendingInteractionOutcome = {
+    action: 'click',
+    command: 'click',
+    positionals: ['100', '144'],
+    flags: { platform: 'ios' },
+    markedAt: Date.now(),
+    attemptsRemaining: 2,
+    preSignature: [
+      {
+        key: 'open-feed|Open feed||Button||enabled|unselected|hittable|#0',
+        x: 20,
+        y: 120,
+        width: 160,
+        height: 48,
+      },
+    ],
+  };
+
+  mockDispatch
+    .mockResolvedValueOnce({
+      nodes: baselineNodes,
+      backend: 'xctest',
+    })
+    .mockResolvedValueOnce({ clicked: true })
+    .mockResolvedValueOnce({
+      nodes: [
+        {
+          index: 0,
+          depth: 0,
+          type: 'Button',
+          label: 'Back',
+          identifier: 'back',
+          hittable: true,
+          rect: { x: 20, y: 60, width: 90, height: 44 },
+        },
+        {
+          index: 1,
+          depth: 0,
+          type: 'StaticText',
+          label: 'Feed',
+          rect: { x: 20, y: 140, width: 160, height: 48 },
+        },
+      ],
+      backend: 'xctest',
+    });
+
+  const result = await captureSnapshot({
+    device: iosSimulatorDevice,
+    session,
+    flags: { snapshotInteractiveOnly: true },
+    logPath: '/tmp/daemon.log',
+  });
+
+  expect(result.snapshot.nodes).toEqual(
+    expect.arrayContaining([expect.objectContaining({ label: 'Feed' })]),
+  );
+  expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['snapshot', 'click', 'snapshot']);
+  expect(mockDispatch.mock.calls[1]?.[2]).toEqual(['100', '144']);
+  expect(session.pendingInteractionOutcome).toBeUndefined();
+});
+
 test('wait text on Android uses freshness-aware capture instead of one-shot snapshot polling', async () => {
   const sessionStore = makeSessionStore();
   const sessionName = 'android-wait-freshness';