perf: optimize and verify scrollintoview ref path

Author: thymikee

README.md

@@ -130,6 +130,8 @@ agent-device press 300 500 --count 12 --interval-ms 45
 agent-device press 300 500 --count 6 --hold-ms 120 --interval-ms 30 --jitter-px 2
 agent-device press @e5 --count 5 --double-tap
 agent-device swipe 540 1500 540 500 120 --count 8 --pause-ms 30 --pattern ping-pong
+agent-device scrollintoview "Sign in"
+agent-device scrollintoview @e42
 ```
 
 ## Command Index
@@ -180,6 +182,7 @@ Swipe timing:
 - `swipe` accepts optional `durationMs` (default `250`, range `16..10000`).
 - Android uses requested swipe duration directly.
 - iOS uses a safe normalized duration to avoid longpress side effects.
+- `scrollintoview` accepts either plain text or a snapshot ref (`@eN`); ref mode uses geometry-based scrolling.
 
 ## Skills
 Install the automation skills listed in [SKILL.md](skills/agent-device/SKILL.md).

src/daemon/__tests__/scroll-planner.test.ts

@@ -0,0 +1,47 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { type RawSnapshotNode } from '../../utils/snapshot.ts';
+import {
+  buildScrollIntoViewPlan,
+  isRectWithinSafeViewportBand,
+  resolveViewportRect,
+} from '../scroll-planner.ts';
+
+function makeNode(index: number, type: string, rect?: RawSnapshotNode['rect']): RawSnapshotNode {
+  return { index, type, rect };
+}
+
+test('resolveViewportRect picks containing application/window viewport', () => {
+  const targetRect = { x: 20, y: 1700, width: 120, height: 40 };
+  const nodes: RawSnapshotNode[] = [
+    makeNode(0, 'Application', { x: 0, y: 0, width: 390, height: 844 }),
+    makeNode(1, 'Window', { x: 0, y: 0, width: 390, height: 844 }),
+    makeNode(2, 'Cell', targetRect),
+  ];
+  const viewport = resolveViewportRect(nodes, targetRect);
+  assert.deepEqual(viewport, { x: 0, y: 0, width: 390, height: 844 });
+});
+
+test('resolveViewportRect returns null when no valid viewport can be inferred', () => {
+  const targetRect = { x: 20, y: 100, width: 120, height: 40 };
+  const nodes: RawSnapshotNode[] = [makeNode(0, 'Cell', undefined)];
+  const viewport = resolveViewportRect(nodes, targetRect);
+  assert.equal(viewport, null);
+});
+
+test('buildScrollIntoViewPlan computes downward content scroll when target is below safe band', () => {
+  const targetRect = { x: 20, y: 2100, width: 120, height: 40 };
+  const viewportRect = { x: 0, y: 0, width: 390, height: 844 };
+  const plan = buildScrollIntoViewPlan(targetRect, viewportRect);
+  assert.ok(plan);
+  assert.equal(plan?.direction, 'down');
+  assert.ok((plan?.count ?? 0) > 1);
+});
+
+test('buildScrollIntoViewPlan returns null when already in safe viewport band', () => {
+  const targetRect = { x: 20, y: 320, width: 120, height: 40 };
+  const viewportRect = { x: 0, y: 0, width: 390, height: 844 };
+  const plan = buildScrollIntoViewPlan(targetRect, viewportRect);
+  assert.equal(plan, null);
+  assert.equal(isRectWithinSafeViewportBand(targetRect, viewportRect), true);
+});

src/daemon/handlers/__tests__/interaction.test.ts

@@ -185,3 +185,186 @@ test('press coordinates does not treat extra trailing args as selector', async (
   assert.deepEqual(dispatchCalls[0]?.positionals, ['100', '200']);
   assert.equal(sessionStore.get(sessionName)?.actions.length, 1);
 });
+
+test('scrollintoview @ref dispatches geometry-based swipe series', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'Application',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+      },
+      {
+        index: 1,
+        type: 'XCUIElementTypeStaticText',
+        label: 'Far item',
+        rect: { x: 20, y: 2600, width: 120, height: 40 },
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  sessionStore.set(sessionName, session);
+
+  const dispatchCalls: Array<{
+    command: string;
+    positionals: string[];
+    context: Record<string, unknown> | undefined;
+  }> = [];
+  let snapshotCallCount = 0;
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'scrollintoview',
+      positionals: ['@e2'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command, positionals, _out, context) => {
+      if (command === 'snapshot') {
+        snapshotCallCount += 1;
+        return {
+          nodes: [
+            { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
+            { index: 1, type: 'XCUIElementTypeStaticText', label: 'Far item', rect: { x: 20, y: 320, width: 120, height: 40 } },
+          ],
+          backend: 'xctest',
+        };
+      }
+      dispatchCalls.push({ command, positionals, context: context as Record<string, unknown> | undefined });
+      return { ok: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true);
+  assert.equal(snapshotCallCount, 1);
+  assert.equal(dispatchCalls.length, 1);
+  assert.equal(dispatchCalls[0]?.command, 'swipe');
+  assert.equal(dispatchCalls[0]?.positionals.length, 5);
+  assert.equal(dispatchCalls[0]?.context?.pattern, 'one-way');
+  assert.equal(dispatchCalls[0]?.context?.pauseMs, 0);
+  assert.equal(typeof dispatchCalls[0]?.context?.count, 'number');
+  assert.ok((dispatchCalls[0]?.context?.count as number) > 1);
+
+  const stored = sessionStore.get(sessionName);
+  assert.ok(stored);
+  assert.equal(stored?.actions.length, 1);
+  assert.equal(stored?.actions[0]?.command, 'scrollintoview');
+  const result = (stored?.actions[0]?.result ?? {}) as Record<string, unknown>;
+  assert.equal(result.ref, 'e2');
+  assert.equal(result.strategy, 'ref-geometry');
+  assert.equal(result.verified, true);
+});
+
+test('scrollintoview @ref returns immediately when target is already in viewport safe band', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'Application',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+      },
+      {
+        index: 1,
+        type: 'XCUIElementTypeStaticText',
+        label: 'Visible item',
+        rect: { x: 20, y: 320, width: 120, height: 40 },
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  sessionStore.set(sessionName, session);
+
+  const dispatchCalls: Array<{ command: string }> = [];
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'scrollintoview',
+      positionals: ['@e2'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command) => {
+      dispatchCalls.push({ command });
+      return { ok: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true);
+  assert.equal(dispatchCalls.length, 0);
+  if (response.ok) {
+    assert.equal(response.data?.attempts, 0);
+    assert.equal(response.data?.alreadyVisible, true);
+  }
+});
+
+test('scrollintoview @ref fails if target remains outside viewport after scroll', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'Application',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+      },
+      {
+        index: 1,
+        type: 'XCUIElementTypeStaticText',
+        label: 'Far item',
+        rect: { x: 20, y: 2600, width: 120, height: 40 },
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'xctest',
+  };
+  sessionStore.set(sessionName, session);
+
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'scrollintoview',
+      positionals: ['@e2'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command) => {
+      if (command === 'snapshot') {
+        return {
+          nodes: [
+            { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
+            { index: 1, type: 'XCUIElementTypeStaticText', label: 'Far item', rect: { x: 20, y: 2600, width: 120, height: 40 } },
+          ],
+          backend: 'xctest',
+        };
+      }
+      return { ok: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, false);
+  if (!response.ok) {
+    assert.equal(response.error?.code, 'COMMAND_FAILED');
+    assert.match(response.error?.message ?? '', /outside viewport/i);
+  }
+});