perf: speed up scrollintoview ref path

Author: thymikee

README.md

@@ -130,6 +130,8 @@ agent-device press 300 500 --count 12 --interval-ms 45
 agent-device press 300 500 --count 6 --hold-ms 120 --interval-ms 30 --jitter-px 2
 agent-device press @e5 --count 5 --double-tap
 agent-device swipe 540 1500 540 500 120 --count 8 --pause-ms 30 --pattern ping-pong
+agent-device scrollintoview "Sign in"
+agent-device scrollintoview @e42
 ```
 
 ## Command Index
@@ -180,6 +182,7 @@ Swipe timing:
 - `swipe` accepts optional `durationMs` (default `250`, range `16..10000`).
 - Android uses requested swipe duration directly.
 - iOS uses a safe normalized duration to avoid longpress side effects.
+- `scrollintoview` accepts either plain text or a snapshot ref (`@eN`); ref mode uses geometry-based scrolling.
 
 ## Skills
 Install the automation skills listed in [SKILL.md](skills/agent-device/SKILL.md).

src/daemon/handlers/__tests__/interaction.test.ts

@@ -185,3 +185,117 @@ test('press coordinates does not treat extra trailing args as selector', async (
   assert.deepEqual(dispatchCalls[0]?.positionals, ['100', '200']);
   assert.equal(sessionStore.get(sessionName)?.actions.length, 1);
 });
+
+test('scrollintoview @ref dispatches geometry-based swipe series', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'Application',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+      },
+      {
+        index: 1,
+        type: 'XCUIElementTypeStaticText',
+        label: 'Far item',
+        rect: { x: 20, y: 2600, width: 120, height: 40 },
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  sessionStore.set(sessionName, session);
+
+  const dispatchCalls: Array<{
+    command: string;
+    positionals: string[];
+    context: Record<string, unknown> | undefined;
+  }> = [];
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'scrollintoview',
+      positionals: ['@e2'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command, positionals, _out, context) => {
+      dispatchCalls.push({ command, positionals, context: context as Record<string, unknown> | undefined });
+      return { ok: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true);
+  assert.equal(dispatchCalls.length, 1);
+  assert.equal(dispatchCalls[0]?.command, 'swipe');
+  assert.equal(dispatchCalls[0]?.positionals.length, 5);
+  assert.equal(dispatchCalls[0]?.context?.pattern, 'one-way');
+  assert.equal(dispatchCalls[0]?.context?.pauseMs, 0);
+  assert.equal(typeof dispatchCalls[0]?.context?.count, 'number');
+  assert.ok((dispatchCalls[0]?.context?.count as number) > 1);
+
+  const stored = sessionStore.get(sessionName);
+  assert.ok(stored);
+  assert.equal(stored?.actions.length, 1);
+  assert.equal(stored?.actions[0]?.command, 'scrollintoview');
+  const result = (stored?.actions[0]?.result ?? {}) as Record<string, unknown>;
+  assert.equal(result.ref, 'e2');
+  assert.equal(result.strategy, 'ref-geometry');
+});
+
+test('scrollintoview @ref returns immediately when target is already in viewport safe band', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'Application',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+      },
+      {
+        index: 1,
+        type: 'XCUIElementTypeStaticText',
+        label: 'Visible item',
+        rect: { x: 20, y: 320, width: 120, height: 40 },
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  sessionStore.set(sessionName, session);
+
+  const dispatchCalls: Array<{ command: string }> = [];
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'scrollintoview',
+      positionals: ['@e2'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command) => {
+      dispatchCalls.push({ command });
+      return { ok: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true);
+  assert.equal(dispatchCalls.length, 0);
+  if (response.ok) {
+    assert.equal(response.data?.attempts, 0);
+    assert.equal(response.data?.alreadyVisible, true);
+  }
+});

src/daemon/handlers/interaction.ts

@@ -1,6 +1,13 @@
 import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
 import { isCommandSupportedOnDevice } from '../../core/capabilities.ts';
-import { attachRefs, centerOfRect, findNodeByRef, normalizeRef, type RawSnapshotNode } from '../../utils/snapshot.ts';
+import {
+  attachRefs,
+  centerOfRect,
+  findNodeByRef,
+  normalizeRef,
+  type RawSnapshotNode,
+  type Rect,
+} from '../../utils/snapshot.ts';
 import type { DaemonCommandContext } from '../context.ts';
 import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts';
 import { SessionStore } from '../session-store.ts';
@@ -548,6 +555,94 @@ export async function handleInteractionCommands(params: {
     return { ok: true, data: { predicate, pass: true, selector: resolved.selector.raw } };
   }
 
+  if (command === 'scrollintoview') {
+    const session = sessionStore.get(sessionName);
+    if (!session) {
+      return {
+        ok: false,
+        error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' },
+      };
+    }
+    const targetInput = req.positionals?.[0] ?? '';
+    if (!targetInput.startsWith('@')) {
+      return null;
+    }
+    const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('scrollintoview', req.flags);
+    if (invalidRefFlagsResponse) return invalidRefFlagsResponse;
+    if (!session.snapshot) {
+      return { ok: false, error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' } };
+    }
+    const ref = normalizeRef(targetInput);
+    if (!ref) {
+      return {
+        ok: false,
+        error: { code: 'INVALID_ARGS', message: 'scrollintoview requires a ref like @e2' },
+      };
+    }
+    let node = findNodeByRef(session.snapshot.nodes, ref);
+    if (!node?.rect && req.positionals && req.positionals.length > 1) {
+      const fallbackLabel = req.positionals.slice(1).join(' ').trim();
+      if (fallbackLabel.length > 0) {
+        node = findNodeByLabel(session.snapshot.nodes, fallbackLabel);
+      }
+    }
+    if (!node?.rect) {
+      return {
+        ok: false,
+        error: { code: 'COMMAND_FAILED', message: `Ref ${targetInput} not found or has no bounds` },
+      };
+    }
+    const viewportRect = resolveViewportRect(session.snapshot.nodes, node.rect);
+    const plan = buildScrollIntoViewPlan(node.rect, viewportRect);
+    const refLabel = resolveRefLabel(node, session.snapshot.nodes);
+    const selectorChain = buildSelectorChainForNode(node, session.device.platform, { action: 'get' });
+    if (!plan) {
+      sessionStore.recordAction(session, {
+        command,
+        positionals: req.positionals ?? [],
+        flags: req.flags ?? {},
+        result: { ref, attempts: 0, alreadyVisible: true, strategy: 'ref-geometry', refLabel, selectorChain },
+      });
+      return { ok: true, data: { ref, attempts: 0, alreadyVisible: true, strategy: 'ref-geometry' } };
+    }
+    const data = await dispatch(
+      session.device,
+      'swipe',
+      [String(plan.x), String(plan.startY), String(plan.x), String(plan.endY), '60'],
+      req.flags?.out,
+      {
+        ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
+        count: plan.count,
+        pauseMs: 0,
+        pattern: 'one-way',
+      },
+    );
+    sessionStore.recordAction(session, {
+      command,
+      positionals: req.positionals ?? [],
+      flags: req.flags ?? {},
+      result: {
+        ...(data ?? {}),
+        ref,
+        attempts: plan.count,
+        direction: plan.direction,
+        strategy: 'ref-geometry',
+        refLabel,
+        selectorChain,
+      },
+    });
+    return {
+      ok: true,
+      data: {
+        ...(data ?? {}),
+        ref,
+        attempts: plan.count,
+        direction: plan.direction,
+        strategy: 'ref-geometry',
+      },
+    };
+  }
+
   return null;
 }
 
@@ -593,7 +688,7 @@ const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [
 ];
 
 function refSnapshotFlagGuardResponse(
-  command: 'press' | 'fill' | 'get',
+  command: 'press' | 'fill' | 'get' | 'scrollintoview',
   flags: CommandFlags | undefined,
 ): DaemonResponse | null {
   const unsupported = unsupportedRefSnapshotFlags(flags);
@@ -623,3 +718,100 @@ export function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): st
   }
   return unsupported;
 }
+
+function resolveViewportRect(nodes: RawSnapshotNode[], targetRect: Rect): Rect {
+  const targetCenter = centerOfRect(targetRect);
+  const rectNodes = nodes.filter((node) => hasValidRect(node.rect));
+  const viewportNodes = rectNodes.filter((node) => {
+    const type = (node.type ?? '').toLowerCase();
+    return type.includes('application') || type.includes('window');
+  });
+
+  const containingViewport = pickLargestRect(
+    viewportNodes
+      .map((node) => node.rect as Rect)
+      .filter((rect) => containsPoint(rect, targetCenter.x, targetCenter.y)),
+  );
+  if (containingViewport) return containingViewport;
+
+  const viewportFallback = pickLargestRect(viewportNodes.map((node) => node.rect as Rect));
+  if (viewportFallback) return viewportFallback;
+
+  const genericContaining = pickLargestRect(
+    rectNodes
+      .map((node) => node.rect as Rect)
+      .filter((rect) => containsPoint(rect, targetCenter.x, targetCenter.y)),
+  );
+  if (genericContaining) return genericContaining;
+
+  return targetRect;
+}
+
+function buildScrollIntoViewPlan(
+  targetRect: Rect,
+  viewportRect: Rect,
+): { x: number; startY: number; endY: number; count: number; direction: 'up' | 'down' } | null {
+  const viewportHeight = Math.max(1, viewportRect.height);
+  const viewportTop = viewportRect.y;
+  const viewportBottom = viewportRect.y + viewportHeight;
+  const safeTop = viewportTop + viewportHeight * 0.25;
+  const safeBottom = viewportBottom - viewportHeight * 0.25;
+  const targetCenterY = targetRect.y + targetRect.height / 2;
+
+  if (targetCenterY >= safeTop && targetCenterY <= safeBottom) {
+    return null;
+  }
+
+  const x = Math.round(viewportRect.x + viewportRect.width / 2);
+  const dragUpStartY = Math.round(viewportTop + viewportHeight * 0.78);
+  const dragUpEndY = Math.round(viewportTop + viewportHeight * 0.22);
+  const dragDownStartY = dragUpEndY;
+  const dragDownEndY = dragUpStartY;
+  const swipeStepPx = Math.max(1, Math.abs(dragUpStartY - dragUpEndY) * 0.9);
+
+  if (targetCenterY > safeBottom) {
+    const delta = targetCenterY - safeBottom;
+    return {
+      x,
+      startY: dragUpStartY,
+      endY: dragUpEndY,
+      count: clampInt(Math.ceil(delta / swipeStepPx), 1, 50),
+      direction: 'down',
+    };
+  }
+
+  const delta = safeTop - targetCenterY;
+  return {
+    x,
+    startY: dragDownStartY,
+    endY: dragDownEndY,
+    count: clampInt(Math.ceil(delta / swipeStepPx), 1, 50),
+    direction: 'up',
+  };
+}
+
+function hasValidRect(rect: Rect | undefined): rect is Rect {
+  if (!rect) return false;
+  return Number.isFinite(rect.x) && Number.isFinite(rect.y) && Number.isFinite(rect.width) && Number.isFinite(rect.height);
+}
+
+function containsPoint(rect: Rect, x: number, y: number): boolean {
+  return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height;
+}
+
+function pickLargestRect(rects: Rect[]): Rect | null {
+  let best: Rect | null = null;
+  let bestArea = -1;
+  for (const rect of rects) {
+    const area = rect.width * rect.height;
+    if (area > bestArea) {
+      best = rect;
+      bestArea = area;
+    }
+  }
+  return best;
+}
+
+function clampInt(value: number, min: number, max: number): number {
+  return Math.min(max, Math.max(min, Math.round(value)));
+}

src/utils/command-schema.ts

@@ -483,8 +483,10 @@ export const COMMAND_SCHEMAS: Record<string, CommandSchema> = {
     allowedFlags: [],
   },
   scrollintoview: {
-    description: 'Scroll until text appears',
-    positionalArgs: ['text'],
+    usageOverride: 'scrollintoview <text|@ref>',
+    description: 'Scroll until text appears or a snapshot ref is brought into view',
+    positionalArgs: ['target'],
+    allowsExtraPositionals: true,
     allowedFlags: [],
   },
   pinch: {

website/docs/docs/commands.md

@@ -56,6 +56,8 @@ agent-device swipe 540 1500 540 500 120
 agent-device swipe 540 1500 540 500 120 --count 8 --pause-ms 30 --pattern ping-pong
 agent-device longpress 300 500 800
 agent-device scroll down 0.5
+agent-device scrollintoview "Sign in"
+agent-device scrollintoview @e42
 agent-device pinch 2.0          # zoom in 2x (iOS simulator)
 agent-device pinch 0.5 200 400 # zoom out at coordinates (iOS simulator)
 ```
@@ -64,6 +66,7 @@ agent-device pinch 0.5 200 400 # zoom out at coordinates (iOS simulator)
 On Android, `fill` also verifies text and performs one clear-and-retry pass on mismatch.
 `swipe` accepts an optional `durationMs` argument (default `250ms`, range `16..10000`).
 On iOS, swipe timing uses a safe normalized duration to avoid longpress side effects.
+`scrollintoview` accepts plain text or a snapshot ref (`@eN`); ref mode uses geometry-based scrolling.
 `longpress` is supported on iOS and Android.
 `pinch` is iOS simulator-only.