fix: improve maestro text tap targets

thymikee · thymikee · commit 618af90dad83 · 2026-05-28T18:22:31.000+02:00
diff --git a/src/compat/maestro/__tests__/runtime-geometry.test.ts b/src/compat/maestro/__tests__/runtime-geometry.test.ts
@@ -0,0 +1,21 @@
+import { expect, test } from 'vitest';
+import { pointForMaestroTapOnTarget } from '../runtime-geometry.ts';
+
+test('pointForMaestroTapOnTarget biases large scroll-area text containers toward the visible label', () => {
+  const point = pointForMaestroTapOnTarget(
+    {
+      node: {
+        index: 5,
+        ref: 'e5',
+        type: 'scroll-area',
+        label: 'Article',
+        rect: { x: 0, y: 117, width: 402, height: 180 },
+      },
+      rect: { x: 0, y: 117, width: 402, height: 180 },
+      frame: { referenceWidth: 402, referenceHeight: 874 },
+    },
+    true,
+  );
+
+  expect(point).toEqual({ x: 84, y: 141 });
+});
diff --git a/src/compat/maestro/__tests__/runtime-interactions.test.ts b/src/compat/maestro/__tests__/runtime-interactions.test.ts
@@ -0,0 +1,85 @@
+import { expect, test } from 'vitest';
+import type { DaemonRequest, DaemonResponse } from '../../../daemon/types.ts';
+import type { SnapshotState } from '../../../utils/snapshot.ts';
+import { invokeMaestroTapOn } from '../runtime-interactions.ts';
+
+test('invokeMaestroTapOn resolves mutating taps from the current raw snapshot', async () => {
+  const selector =
+    'label="Article by Gandalf" || text="Article by Gandalf" || id="Article by Gandalf"';
+
+  const clicks: string[][] = [];
+  let snapshots = 0;
+  const response = await invokeMaestroTapOn({
+    baseReq: {
+      token: 'test',
+      session: 'nav',
+      flags: { platform: 'ios' },
+    },
+    positionals: [selector],
+    invoke: async (req: DaemonRequest): Promise<DaemonResponse> => {
+      if (req.command === 'snapshot') {
+        snapshots += 1;
+        return { ok: true, data: currentBreadcrumbSnapshot() };
+      }
+      if (req.command === 'click') {
+        clicks.push(req.positionals ?? []);
+        return { ok: true, data: {} };
+      }
+      return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } };
+    },
+  });
+
+  expect(response.ok).toBe(true);
+  expect(snapshots).toBe(1);
+  expect(clicks).toEqual([['86', '89']]);
+});
+
+function currentBreadcrumbSnapshot(): SnapshotState {
+  return {
+    createdAt: Date.now(),
+    nodes: [
+      appNode(),
+      windowNode(),
+      {
+        index: 2,
+        ref: 'e3',
+        type: 'ScrollView',
+        label: 'Article by Gandalf',
+        depth: 4,
+        parentIndex: 1,
+        rect: { x: 0, y: 58.33333333333333, width: 402, height: 58.33333333333333 },
+      },
+      {
+        index: 3,
+        ref: 'e4',
+        type: 'Cell',
+        label: 'Article by Gandalf',
+        depth: 5,
+        parentIndex: 2,
+        rect: { x: 8, y: 65.33333587646484, width: 155, height: 48 },
+      },
+    ],
+  };
+}
+
+function appNode(): SnapshotState['nodes'][number] {
+  return {
+    index: 0,
+    ref: 'e1',
+    type: 'Application',
+    label: 'React Navigation Example',
+    depth: 0,
+    rect: { x: 0, y: 0, width: 402, height: 874 },
+  };
+}
+
+function windowNode(): SnapshotState['nodes'][number] {
+  return {
+    index: 1,
+    ref: 'e2',
+    type: 'Window',
+    depth: 1,
+    parentIndex: 0,
+    rect: { x: 0, y: 0, width: 402, height: 874 },
+  };
+}
diff --git a/src/compat/maestro/__tests__/runtime-targets.test.ts b/src/compat/maestro/__tests__/runtime-targets.test.ts
@@ -161,6 +161,95 @@ test('resolveVisibleMaestroNodeFromSnapshot requires visible text matches to be
   });
 });
 
+test('resolveMaestroNodeFromSnapshot infers missing selected tab slot from tab-strip children', () => {
+  const snapshot: SnapshotState = {
+    createdAt: Date.now(),
+    nodes: [
+      {
+        index: 1,
+        ref: 'e1',
+        type: 'ScrollView',
+        label: 'Chat',
+        rect: { x: 0, y: 116.66666412353516, width: 402, height: 48 },
+        depth: 3,
+      },
+      {
+        index: 2,
+        ref: 'e2',
+        type: 'Cell',
+        label: 'Contacts',
+        rect: { x: 134, y: 116.66666412353516, width: 134, height: 48 },
+        depth: 4,
+        parentIndex: 1,
+      },
+      {
+        index: 3,
+        ref: 'e3',
+        type: 'Cell',
+        label: 'Albums',
+        rect: { x: 268, y: 116.66666412353516, width: 134, height: 48 },
+        depth: 4,
+        parentIndex: 1,
+      },
+    ],
+  };
+
+  const target = resolveMaestroNodeFromSnapshot(
+    snapshot,
+    'label="Chat" || text="Chat" || id="Chat"',
+    {},
+    'ios',
+    { referenceWidth: 402, referenceHeight: 874 },
+    { promoteTapTarget: true },
+  );
+
+  expect(target).toMatchObject({
+    ok: true,
+    node: expect.objectContaining({ index: 1 }),
+    rect: { x: 0, y: 116.66666412353516, width: 134, height: 48 },
+  });
+});
+
+test('resolveMaestroNodeFromSnapshot keeps concrete child matches over tab-strip inference', () => {
+  const snapshot: SnapshotState = {
+    createdAt: Date.now(),
+    nodes: [
+      {
+        index: 1,
+        ref: 'e1',
+        type: 'ScrollView',
+        label: 'Article by Gandalf',
+        rect: { x: 0, y: 58.33333333333333, width: 402, height: 58.33333333333333 },
+        depth: 4,
+      },
+      {
+        index: 2,
+        ref: 'e2',
+        type: 'Cell',
+        label: 'Article by Gandalf',
+        rect: { x: 8, y: 65.33333587646484, width: 155, height: 48 },
+        depth: 5,
+        parentIndex: 1,
+      },
+    ],
+  };
+
+  const target = resolveMaestroNodeFromSnapshot(
+    snapshot,
+    'label="Article by Gandalf" || text="Article by Gandalf" || id="Article by Gandalf"',
+    {},
+    'ios',
+    { referenceWidth: 402, referenceHeight: 874 },
+    { promoteTapTarget: true },
+  );
+
+  expect(target).toMatchObject({
+    ok: true,
+    node: expect.objectContaining({ index: 2 }),
+    rect: { x: 8, y: 65.33333587646484, width: 155, height: 48 },
+  });
+});
+
 function makeReactNativeOverlaySnapshot(): SnapshotState {
   return {
     createdAt: Date.now(),
diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts
@@ -6,7 +6,6 @@ import { sleep } from '../../utils/timeouts.ts';
 import {
   captureMaestroRawSnapshot,
   errorResponse,
-  rememberMaestroSnapshot,
   readSnapshotState,
   type MaestroRuntimeInvoke,
   type ReplayBaseRequest,
@@ -57,7 +56,6 @@ export async function invokeMaestroAssertVisible(params: {
         getSnapshotReferenceFrame(snapshot),
       );
       if (target.ok) {
-        rememberMaestroSnapshot(params.scope, response.data, selector);
         return {
           ok: true,
           data: {
diff --git a/src/compat/maestro/runtime-geometry.ts b/src/compat/maestro/runtime-geometry.ts
@@ -1,4 +1,5 @@
 import type { Rect, SnapshotNode } from '../../utils/snapshot.ts';
+import { normalizeType } from '../../utils/snapshot-processing.ts';
 import type { MaestroSnapshotTarget } from './runtime-targets.ts';
 
 const MAESTRO_GEOMETRY_POLICY = {
@@ -108,14 +109,13 @@ function shouldBiasMaestroVisibleTextTap(
   rect: Rect,
 ): boolean {
   if (!isVisibleTextSelector) return false;
-  if (
-    rect.height < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minHeight ||
-    rect.width < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minWidth
-  ) {
+  if (rect.width < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minWidth) {
     return false;
   }
-  const type = node.type?.toLowerCase();
-  return type === 'cell' || type === 'other' || type === 'scrollview';
+  const type = normalizeType(node.type ?? '');
+  const scrollableTextContainer = type === 'scrollview' || type === 'scroll-area';
+  if (rect.height < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minHeight) return false;
+  return type === 'cell' || type === 'other' || scrollableTextContainer;
 }
 
 function interiorCoordinate(origin: number, size: number): number {
diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts
@@ -5,7 +5,6 @@ import { sleep } from '../../utils/timeouts.ts';
 import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts';
 import {
   captureMaestroRawSnapshot,
-  consumeMaestroSnapshot,
   errorResponse,
   readCachedMaestroReferenceFrame,
   readSnapshotState,
@@ -471,14 +470,6 @@ async function resolveMaestroSnapshotTarget(
   commandLabel: string,
   resolutionOptions: { promoteTapTarget: boolean },
 ): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> {
-  const cachedTarget = resolveCachedMaestroSnapshotTarget(
-    params,
-    selector,
-    options,
-    resolutionOptions,
-  );
-  if (cachedTarget.ok) return cachedTarget;
-
   const snapshotResponse = await captureMaestroRawSnapshot(params);
   if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse };
 
@@ -543,37 +534,6 @@ async function resolveMaestroSnapshotTarget(
   };
 }
 
-function resolveCachedMaestroSnapshotTarget(
-  params: {
-    baseReq: ReplayBaseRequest;
-    scope?: ReplayVarScope;
-  },
-  selector: string,
-  options: MaestroTapOnOptions,
-  resolutionOptions: { promoteTapTarget: boolean },
-): { ok: true; target: MaestroSnapshotTarget } | { ok: false } {
-  const cached = consumeMaestroSnapshot(params.scope, selector);
-  if (!cached) return { ok: false };
-  const resolution = resolveMaestroNodeFromSnapshot(
-    cached.snapshot,
-    selector,
-    options,
-    readMaestroSelectorPlatform(params.baseReq.flags),
-    cached.frame,
-    resolutionOptions,
-  );
-  return resolution.ok
-    ? {
-        ok: true,
-        target: {
-          node: resolution.node,
-          rect: resolution.rect,
-          frame: cached.frame,
-        },
-      }
-    : { ok: false };
-}
-
 function readMaestroTapOnOptions(
   rawOptions: string | undefined,
 ): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } {
diff --git a/src/compat/maestro/runtime-support.ts b/src/compat/maestro/runtime-support.ts
@@ -20,10 +20,6 @@ export type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise<DaemonRespons
 export type FailedDaemonResponse = Extract<DaemonResponse, { ok: false }>;
 
 const maestroReferenceFrameCache = new WeakMap<ReplayVarScope, TouchReferenceFrame>();
-const maestroSnapshotCache = new WeakMap<
-  ReplayVarScope,
-  { snapshot: SnapshotState; frame: TouchReferenceFrame | undefined; selector: string }
->();
 
 export function errorResponse(
   code: string,
@@ -73,31 +69,6 @@ export function readCachedMaestroReferenceFrame(
   return scope ? maestroReferenceFrameCache.get(scope) : undefined;
 }
 
-export function rememberMaestroSnapshot(
-  scope: ReplayVarScope | undefined,
-  data: unknown,
-  selector: string,
-): void {
-  if (!scope) return;
-  const snapshot = readSnapshotState(data);
-  if (!snapshot) return;
-  maestroSnapshotCache.set(scope, {
-    snapshot,
-    frame: getSnapshotReferenceFrame(snapshot),
-    selector,
-  });
-}
-
-export function consumeMaestroSnapshot(
-  scope: ReplayVarScope | undefined,
-  selector: string,
-): { snapshot: SnapshotState; frame: TouchReferenceFrame | undefined } | undefined {
-  if (!scope) return undefined;
-  const cached = maestroSnapshotCache.get(scope);
-  maestroSnapshotCache.delete(scope);
-  return cached?.selector === selector ? cached : undefined;
-}
-
 function rememberMaestroReferenceFrame(scope: ReplayVarScope, data: unknown): void {
   const snapshot = readSnapshotState(data);
   const frame = getSnapshotReferenceFrame(snapshot);
diff --git a/src/compat/maestro/runtime-targets.ts b/src/compat/maestro/runtime-targets.ts
diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts
