fix: return deterministic matched-target metadata for find click responses (#178)

* fix: return deterministic matched-target metadata for find click responses (#170)

On success, build the response from the matched snapshot node rather than
passing through whatever the underlying click/press handler returns.
Response now includes: ref, locator, query, and x/y derived from the
matched element's rect — all stable across runs.

Also makes `dispatch` injectable in `handleFindCommands` (matching the
pattern in `handleInteractionCommands`) and adds tests that assert the
deterministic shape and verify that non-deterministic platform runner
data does not bleed into the response.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* docs: document find click response shape in selectors and skill references

Add response shape documentation for `find "<query>" click --json` to:
- website/docs/docs/selectors.md (new "Response shape (click)" section)
- skills/agent-device/SKILL.md (guardrail note)
- skills/agent-device/references/snapshot-refs.md (new "find click response" section)

Follows the fix in #178 that made the response deterministic.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: thymikee

skills/agent-device/SKILL.md

@@ -170,6 +170,7 @@ agent-device batch --steps-file /tmp/batch-steps.json --json
 - Re-snapshot after UI mutations (navigation/modal/list changes).
 - Prefer `snapshot -i`; scope/depth only when needed.
 - Use refs for discovery, selectors for replay/assertions.
+- `find "<query>" click --json` returns `{ ref, locator, query, x, y }` — all derived from the matched snapshot node. Do not rely on these fields from raw `press`/`click` responses for observability; use `find` instead.
 - Use `fill` for clear-then-type semantics; use `type` for focused append typing.
 - Use `install` for in-place app upgrades (keep app data when platform permits), and `reinstall` for deterministic fresh-state runs.
 - App binary format support for `install`/`reinstall`: Android `.apk`/`.aab`, iOS `.app`/`.ipa`.

skills/agent-device/references/snapshot-refs.md

@@ -76,6 +76,16 @@ Efficient pattern:
 
 - If refs are unstable after UI transitions, switch to selector-based targeting and stop investing in ref-only flows.
 
+## find click response
+
+`find "<query>" click --json` returns deterministic matched-target metadata:
+
+```json
+{ "ref": "@e3", "locator": "any", "query": "Increment", "x": 195, "y": 422 }
+```
+
+Fields come from the matched snapshot node, not the platform runner. Use these for observability and replay quality — they are stable across runs for the same UI state.
+
 ## Replay note
 
 - Prefer selector-based actions in recorded `.ad` replays.

src/daemon/handlers/__tests__/find.test.ts

@@ -1,7 +1,41 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
-import { parseFindArgs } from '../find.ts';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { parseFindArgs, handleFindCommands } from '../find.ts';
 import { AppError } from '../../../utils/errors.ts';
+import { SessionStore } from '../../session-store.ts';
+import type { SessionState } from '../../types.ts';
+import type { DaemonRequest, DaemonResponse } from '../../types.ts';
+
+function makeSessionStore(): SessionStore {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-find-handler-'));
+  return new SessionStore(path.join(root, 'sessions'));
+}
+
+function makeSession(name: string): SessionState {
+  return {
+    name,
+    device: {
+      platform: 'ios',
+      id: 'sim-1',
+      name: 'iPhone 17 Pro',
+      kind: 'simulator',
+      booted: true,
+    },
+    createdAt: Date.now(),
+    actions: [],
+  };
+}
+
+const INCREMENT_NODE = {
+  type: 'Button',
+  label: 'Increment',
+  hittable: true,
+  rect: { x: 50, y: 0, width: 100, height: 100 },
+  depth: 0,
+};
 
 test('parseFindArgs defaults to click with any locator', () => {
   const parsed = parseFindArgs(['Login']);
@@ -97,3 +131,81 @@ test('parseFindArgs with bare locator yields empty query', () => {
   assert.equal(parsed.query, '');
   assert.equal(parsed.action, 'click');
 });
+
+test('handleFindCommands click returns deterministic matched-target metadata', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  sessionStore.set(sessionName, makeSession(sessionName));
+
+  const invokeCalls: DaemonRequest[] = [];
+  const response = await handleFindCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'find',
+      positionals: ['Increment', 'click'],
+      flags: {},
+    },
+    sessionName,
+    logPath: '/tmp/test.log',
+    sessionStore,
+    invoke: async (req) => {
+      invokeCalls.push(req);
+      // Simulate runner returning non-deterministic platform data that should not bleed through
+      return { ok: true, data: { platformSpecificRef: 'XCUIElementTypeApplication', x: 0, y: 0 } };
+    },
+    dispatch: async (_device, command) => {
+      if (command === 'snapshot') {
+        return { nodes: [INCREMENT_NODE] };
+      }
+      return {};
+    },
+  });
+
+  assert.ok(response, 'expected a response');
+  assert.ok(response.ok, 'expected success');
+  const data = response.data as Record<string, unknown>;
+
+  // Deterministic matched-target metadata
+  assert.equal(data.ref, '@e1', 'ref must match the resolved snapshot node');
+  assert.equal(data.locator, 'any', 'locator must reflect the find strategy');
+  assert.equal(data.query, 'Increment', 'query must reflect the search term');
+  assert.equal(data.x, 100, 'x must be derived from the matched node rect center');
+  assert.equal(data.y, 50, 'y must be derived from the matched node rect center');
+
+  // Non-deterministic platform data must not leak through
+  assert.equal(data.platformSpecificRef, undefined, 'platform runner data must not appear in response');
+
+  // invoke was called with the resolved ref
+  assert.equal(invokeCalls.length, 1);
+  assert.equal(invokeCalls[0].positionals?.[0], '@e1');
+});
+
+test('handleFindCommands click with explicit label locator returns locator in metadata', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  sessionStore.set(sessionName, makeSession(sessionName));
+
+  const response = await handleFindCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'find',
+      positionals: ['label', 'Increment', 'click'],
+      flags: {},
+    },
+    sessionName,
+    logPath: '/tmp/test.log',
+    sessionStore,
+    invoke: async () => ({ ok: true, data: {} }),
+    dispatch: async (_device, command) => {
+      if (command === 'snapshot') return { nodes: [INCREMENT_NODE] };
+      return {};
+    },
+  });
+
+  assert.ok(response?.ok);
+  const data = response!.data as Record<string, unknown>;
+  assert.equal(data.locator, 'label');
+  assert.equal(data.query, 'Increment');
+});

src/daemon/handlers/find.ts

@@ -19,8 +19,10 @@ export async function handleFindCommands(params: {
   logPath: string;
   sessionStore: SessionStore;
   invoke: (req: DaemonRequest) => Promise<DaemonResponse>;
+  dispatch?: typeof dispatchCommand;
 }): Promise<DaemonResponse | null> {
   const { req, sessionName, logPath, sessionStore, invoke } = params;
+  const dispatch = params.dispatch ?? dispatchCommand;
   const command = req.command;
   if (command !== 'find') return null;
 
@@ -59,7 +61,7 @@ export async function handleFindCommands(params: {
     if (lastNodes && now - lastSnapshotAt < 750) {
       return { nodes: lastNodes };
     }
-    const data = (await dispatchCommand(device, 'snapshot', [], req.flags?.out, {
+    const data = (await dispatch(device, 'snapshot', [], req.flags?.out, {
       ...contextFromFlags(
         logPath,
         {
@@ -187,15 +189,21 @@ export async function handleFindCommands(params: {
       flags: actionFlags,
     });
     if (!response.ok) return response;
+    const matchCoords = resolvedNode.rect ? centerOfRect(resolvedNode.rect) : null;
+    const matchData: Record<string, unknown> = { ref, locator, query };
+    if (matchCoords) {
+      matchData.x = matchCoords.x;
+      matchData.y = matchCoords.y;
+    }
     if (session) {
       sessionStore.recordAction(session, {
         command,
         positionals: req.positionals ?? [],
         flags: req.flags ?? {},
-        result: { ref, action: 'click' },
+        result: { ref, action: 'click', locator, query },
       });
     }
-    return response;
+    return { ok: true, data: matchData };
   }
   if (action === 'fill') {
     if (!value) {
@@ -224,7 +232,7 @@ export async function handleFindCommands(params: {
     if (!coords) {
       return { ok: false, error: { code: 'COMMAND_FAILED', message: 'matched element has no bounds' } };
     }
-    const response = await dispatchCommand(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, {
+    const response = await dispatch(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, {
       ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath),
     });
     if (session) {
@@ -245,10 +253,10 @@ export async function handleFindCommands(params: {
     if (!coords) {
       return { ok: false, error: { code: 'COMMAND_FAILED', message: 'matched element has no bounds' } };
     }
-    await dispatchCommand(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, {
+    await dispatch(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, {
       ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath),
     });
-    const response = await dispatchCommand(device, 'type', [value], req.flags?.out, {
+    const response = await dispatch(device, 'type', [value], req.flags?.out, {
       ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath),
     });
     if (session) {

website/docs/docs/selectors.md

@@ -20,3 +20,22 @@ Tips:
 - Use `find ... wait <timeoutMs>` to wait for UI to appear.
 - Combine with scoped snapshots using `snapshot -s "<label>"` for speed.
 - [Android] If a matched node is not hittable, agent-device will click/focus the nearest hittable ancestor.
+
+## Response shape (click)
+
+`find "<query>" click --json` returns deterministic matched-target metadata derived from the resolved snapshot node:
+
+```json
+{
+  "ref": "@e3",
+  "locator": "any",
+  "query": "Sign In",
+  "x": 195,
+  "y": 422
+}
+```
+
+- `ref` — snapshot ref of the matched (or nearest hittable ancestor) element.
+- `locator` — find strategy used (`any`, `text`, `label`, `value`, `role`, `id`).
+- `query` — the search term as provided.
+- `x`, `y` — tap coordinates derived from the matched element's rect center.