fix: improve maestro tab target swipes

Author: thymikee

src/compat/maestro/__tests__/runtime-interactions.test.ts

@@ -1,7 +1,7 @@
 import { expect, test } from 'vitest';
 import type { DaemonRequest, DaemonResponse } from '../../../daemon/types.ts';
 import type { SnapshotState } from '../../../utils/snapshot.ts';
-import { invokeMaestroTapOn } from '../runtime-interactions.ts';
+import { invokeMaestroSwipeScreen, invokeMaestroTapOn } from '../runtime-interactions.ts';
 
 test('invokeMaestroTapOn resolves mutating taps from the current raw snapshot', async () => {
   const selector =
@@ -34,6 +34,31 @@ test('invokeMaestroTapOn resolves mutating taps from the current raw snapshot',
   expect(clicks).toEqual([['86', '89']]);
 });
 
+test('invokeMaestroSwipeScreen uses a conservative Android content-lane directional swipe', async () => {
+  const swipes: string[][] = [];
+  const response = await invokeMaestroSwipeScreen({
+    baseReq: {
+      token: 'test',
+      session: 'pager',
+      flags: { platform: 'android' },
+    },
+    positionals: ['direction', 'left', '300'],
+    invoke: async (req: DaemonRequest): Promise<DaemonResponse> => {
+      if (req.command === 'snapshot') {
+        return { ok: true, data: fullScreenSnapshot(1080, 2340) };
+      }
+      if (req.command === 'swipe') {
+        swipes.push(req.positionals ?? []);
+        return { ok: true, data: {} };
+      }
+      return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } };
+    },
+  });
+
+  expect(response.ok).toBe(true);
+  expect(swipes).toEqual([['756', '1521', '324', '1521', '300']]);
+});
+
 function currentBreadcrumbSnapshot(): SnapshotState {
   return {
     createdAt: Date.now(),
@@ -62,6 +87,30 @@ function currentBreadcrumbSnapshot(): SnapshotState {
   };
 }
 
+function fullScreenSnapshot(width: number, height: number): SnapshotState {
+  return {
+    createdAt: Date.now(),
+    nodes: [
+      {
+        index: 0,
+        ref: 'e1',
+        type: 'Application',
+        label: 'Android Test App',
+        depth: 0,
+        rect: { x: 0, y: 0, width, height },
+      },
+      {
+        index: 1,
+        ref: 'e2',
+        type: 'Window',
+        depth: 1,
+        parentIndex: 0,
+        rect: { x: 0, y: 0, width, height },
+      },
+    ],
+  };
+}
+
 function appNode(): SnapshotState['nodes'][number] {
   return {
     index: 0,

src/compat/maestro/__tests__/runtime-targets.test.ts

@@ -250,6 +250,55 @@ test('resolveMaestroNodeFromSnapshot keeps concrete child matches over tab-strip
   });
 });
 
+test('resolveMaestroNodeFromSnapshot infers leading breadcrumb slot when selected child is omitted', () => {
+  const snapshot: SnapshotState = {
+    createdAt: Date.now(),
+    nodes: [
+      {
+        index: 1,
+        ref: 'e1',
+        type: 'ScrollView',
+        label: 'Article by Gandalf',
+        rect: { x: 0, y: 58.33333333333333, width: 402, height: 58.33333333333333 },
+        depth: 4,
+      },
+      {
+        index: 2,
+        ref: 'e2',
+        type: 'Other',
+        label: 'Feed',
+        rect: { x: 170.3333282470703, y: 65.33333587646484, width: 54, height: 48 },
+        depth: 5,
+        parentIndex: 1,
+      },
+      {
+        index: 3,
+        ref: 'e3',
+        type: 'Other',
+        label: 'Albums',
+        rect: { x: 231.6666717529297, y: 65.33333587646484, width: 75, height: 48 },
+        depth: 5,
+        parentIndex: 1,
+      },
+    ],
+  };
+
+  const target = resolveMaestroNodeFromSnapshot(
+    snapshot,
+    'label="Article by Gandalf" || text="Article by Gandalf" || id="Article by Gandalf"',
+    {},
+    'ios',
+    { referenceWidth: 402, referenceHeight: 874 },
+    { promoteTapTarget: true },
+  );
+
+  expect(target).toMatchObject({
+    ok: true,
+    node: expect.objectContaining({ index: 1 }),
+    rect: { x: 0, y: 58.33333333333333, width: 168, height: 58.33333333333333 },
+  });
+});
+
 function makeReactNativeOverlaySnapshot(): SnapshotState {
   return {
     createdAt: Date.now(),

src/compat/maestro/runtime-interactions.ts

@@ -270,12 +270,14 @@ function resolveDirectionalScreenSwipe(
     case 'down':
       return { ok: true, start: point(50, 20), end: point(50, 80), durationMs };
     case 'left': {
-      const yPercent = androidHorizontalContentSwipeY(platform, 80, 50, 20, 50);
-      return { ok: true, start: point(80, yPercent), end: point(20, yPercent), durationMs };
+      const [startX, endX] = androidHorizontalDirectionalSwipeX(platform, 80, 20);
+      const yPercent = androidHorizontalContentSwipeY(platform, startX, 50, endX, 50);
+      return { ok: true, start: point(startX, yPercent), end: point(endX, yPercent), durationMs };
     }
     case 'right': {
-      const yPercent = androidHorizontalContentSwipeY(platform, 20, 50, 80, 50);
-      return { ok: true, start: point(20, yPercent), end: point(80, yPercent), durationMs };
+      const [startX, endX] = androidHorizontalDirectionalSwipeX(platform, 20, 80);
+      const yPercent = androidHorizontalContentSwipeY(platform, startX, 50, endX, 50);
+      return { ok: true, start: point(startX, yPercent), end: point(endX, yPercent), durationMs };
     }
     default:
       return {
@@ -288,6 +290,15 @@ function resolveDirectionalScreenSwipe(
   }
 }
 
+function androidHorizontalDirectionalSwipeX(
+  platform: string,
+  startX: number,
+  endX: number,
+): [number, number] {
+  if (platform !== 'android') return [startX, endX];
+  return startX < endX ? [30, 70] : [70, 30];
+}
+
 function resolvePercentScreenSwipe(
   args: string[],
   frame: { referenceWidth: number; referenceHeight: number },
@@ -320,7 +331,7 @@ function androidHorizontalContentSwipeY(
 ): number {
   if (platform !== 'android') return y2;
   if (y1 !== y2 || y1 !== 50) return y2;
-  if (Math.abs(x2 - x1) < 50) return y2;
+  if (Math.abs(x2 - x1) < 30) return y2;
   // Maestro's Android driver treats 50% horizontal swipes as content swipes.
   // Raw `adb input swipe` at the physical screen midpoint can land above
   // horizontally paged content in React Native layouts, so use a lower content

src/compat/maestro/runtime-targets.ts

@@ -476,24 +476,88 @@ function inferMaestroMissingTabSlotMatch(
   query: string,
 ): MaestroResolvedSnapshotMatch | null {
   if (!isMaestroTabStripContainerMatch(match, query)) return null;
-  const children: Array<SnapshotNode & { rect: Rect }> = [];
-  for (const node of nodes) {
-    if (node.parentIndex !== match.node.index || !node.rect) continue;
-    const candidate = node as SnapshotNode & { rect: Rect };
-    if (isMaestroTabStripChildCandidate(candidate, match.rect, query)) {
-      children.push(candidate);
-    }
-  }
-  children.sort((left, right) => left.rect.x - right.rect.x);
+  const children = collectMaestroTabStripChildCandidates(nodes, match, query);
   if (children.length === 0) return null;
   const medianChildWidth = median(children.map((child) => child.rect.width));
-  const gaps = resolveHorizontalGaps(
+  const allGaps = resolveHorizontalGaps(
     match.rect,
     children.map((child) => child.rect),
-  ).filter((gap) => isPlausibleMissingTabSlot(gap.width, medianChildWidth));
-  if (gaps.length !== 1) return null;
-  const gap = gaps[0];
+  );
+  const gap = selectMaestroMissingSlotGap(match, query, allGaps, medianChildWidth);
   if (!gap) return null;
+  return matchWithRect(match, gap);
+}
+
+function collectMaestroTabStripChildCandidates(
+  nodes: SnapshotState['nodes'],
+  match: MaestroResolvedSnapshotMatch,
+  query: string,
+): Array<SnapshotNode & { rect: Rect }> {
+  return nodes
+    .filter((node): node is SnapshotNode & { rect: Rect } => {
+      return (
+        node.parentIndex === match.node.index &&
+        Boolean(node.rect) &&
+        isMaestroTabStripChildCandidate(node as SnapshotNode & { rect: Rect }, match.rect, query)
+      );
+    })
+    .sort((left, right) => left.rect.x - right.rect.x);
+}
+
+function selectMaestroMissingSlotGap(
+  match: MaestroResolvedSnapshotMatch,
+  query: string,
+  gaps: Array<{ x: number; width: number }>,
+  medianChildWidth: number,
+): { x: number; width: number } | null {
+  const plausibleGaps = gaps.filter((gap) => isPlausibleMissingTabSlot(gap.width, medianChildWidth));
+  const leadingTextSlot = inferMaestroLeadingTextSlotGap(match, query, gaps);
+  const hasPlausibleLeadingGap = plausibleGaps.some((gap) => isLeadingGap(match.rect, gap));
+  if (leadingTextSlot && !hasPlausibleLeadingGap) return leadingTextSlot;
+  if (plausibleGaps.length === 1) return plausibleGaps[0] ?? null;
+  return leadingTextSlot;
+}
+
+function inferMaestroLeadingTextSlotGap(
+  match: MaestroResolvedSnapshotMatch,
+  query: string,
+  gaps: Array<{ x: number; width: number }>,
+): { x: number; width: number } | null {
+  const leadingGap = gaps.find((gap) => Math.abs(gap.x - match.rect.x) < 1);
+  const estimatedLabelWidth = Math.max(48, Math.min(220, query.length * 8 + 24));
+  if (!isLeadingTextSlotCandidate(match, query, leadingGap, estimatedLabelWidth)) return null;
+  return {
+    x: match.rect.x,
+    width: Math.min(estimatedLabelWidth, leadingGap.width),
+  };
+}
+
+function isLeadingTextSlotCandidate(
+  match: MaestroResolvedSnapshotMatch,
+  query: string,
+  gap: { x: number; width: number } | undefined,
+  estimatedLabelWidth: number,
+): gap is { x: number; width: number } {
+  if (!gap) return false;
+  return (
+    normalizeType(match.node.type ?? '') === 'scrollview' &&
+    maestroVisibleTextMatchRank(match.node, query) <= 1 &&
+    match.rect.width >= 240 &&
+    match.rect.height >= 32 &&
+    match.rect.height <= 80 &&
+    gap.width <= match.rect.width * 0.55 &&
+    gap.width >= estimatedLabelWidth * 0.6
+  );
+}
+
+function isLeadingGap(rect: Rect, gap: { x: number; width: number }): boolean {
+  return Math.abs(gap.x - rect.x) < 1;
+}
+
+function matchWithRect(
+  match: MaestroResolvedSnapshotMatch,
+  gap: { x: number; width: number },
+): MaestroResolvedSnapshotMatch {
   return {
     ...match,
     rect: {

src/daemon/handlers/__tests__/session-replay-vars.test.ts

@@ -1473,7 +1473,7 @@ test('runReplayScriptFile uses Android content lane for Maestro horizontal scree
     calls.map((call) => [call.command, call.positionals]),
     [
       ['snapshot', []],
-      ['swipe', ['320', '520', '80', '520', '300']],
+      ['swipe', ['280', '520', '120', '520', '300']],
       ['swipe', ['360', '520', '40', '520', '300']],
     ],
   );