Skip to content

Commit 83042cc

Browse files
thymikeethymikee
authored
fix: preserve scoped Android snapshot refs (#456)
1 parent 7c5b767 commit 83042cc

15 files changed

Lines changed: 404 additions & 25 deletions

‎src/daemon/handlers/__tests__/find.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,9 +226,10 @@ test('handleFindCommands click returns deterministic metadata across locator var
226226
positionals: ['Increment', 'click'],
227227
nodes: [hittableParentNoRect, nonHittableChildWithRect],
228228
invoke: async () => ({ platformSpecificRef: 'XCUIElementTypeView' }),
229-
expectedKeys: ['locator', 'query', 'ref'],
229+
expectedKeys: ['locator', 'query', 'ref', 'x', 'y'],
230230
expectedLocator: 'any',
231231
expectedQuery: 'Increment',
232+
expectedCoordinates: { x: 100, y: 50 },
232233
},
233234
{
234235
label: 'keeps explicit label locator in metadata',

‎src/daemon/handlers/__tests__/replay-heal.test.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import os from 'node:os';
1111
import path from 'node:path';
1212
import type { CommandFlags } from '../../../core/dispatch.ts';
1313
import { handleSessionCommands } from '../session.ts';
14+
import { healReplayAction } from '../session-replay-heal.ts';
1415
import { SessionStore } from '../../session-store.ts';
1516
import type { DaemonRequest, DaemonResponse, SessionAction } from '../../types.ts';
1617
import type { DeviceInfo } from '../../../utils/device.ts';
@@ -89,6 +90,59 @@ function tokenizeReplayLine(line: string): string[] {
8990
return tokens;
9091
}
9192

93+
test('replay heal snapshot refresh clears stale scoped snapshot source', async () => {
94+
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-scope-source-'));
95+
const sessionsDir = path.join(tempRoot, 'sessions');
96+
const sessionStore = new SessionStore(sessionsDir);
97+
const sessionName = 'heal-scope-source-session';
98+
const session = makeSession(sessionName);
99+
session.snapshotScopeSource = {
100+
nodes: [
101+
{
102+
ref: 'e1',
103+
index: 0,
104+
depth: 0,
105+
type: 'Button',
106+
label: 'Stale button',
107+
},
108+
],
109+
createdAt: Date.now(),
110+
backend: 'xctest',
111+
};
112+
sessionStore.set(sessionName, session);
113+
114+
mockDispatchCommand.mockResolvedValue({
115+
nodes: [
116+
{
117+
index: 0,
118+
depth: 0,
119+
type: 'Button',
120+
label: 'Continue',
121+
rect: { x: 0, y: 0, width: 100, height: 44 },
122+
hittable: true,
123+
},
124+
],
125+
truncated: false,
126+
backend: 'xctest',
127+
});
128+
129+
const healed = await healReplayAction({
130+
action: {
131+
ts: Date.now(),
132+
command: 'click',
133+
positionals: ['label="Continue"'],
134+
flags: {},
135+
result: {},
136+
},
137+
sessionName,
138+
logPath: '/tmp/replay.log',
139+
sessionStore,
140+
});
141+
142+
expect(healed?.positionals[0]).toContain('label="Continue"');
143+
expect(sessionStore.get(sessionName)?.snapshotScopeSource).toBeUndefined();
144+
});
145+
92146
test('replay --update heals selector and rewrites replay file', async () => {
93147
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-'));
94148
const sessionsDir = path.join(tempRoot, 'sessions');

‎src/daemon/handlers/__tests__/snapshot-scoped-refs.test.ts‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import { beforeEach, expect, test, vi } from 'vitest';
2+
import { handleSnapshotCommands } from '../snapshot.ts';
3+
import type { RawSnapshotNode, SnapshotState } from '../../../utils/snapshot.ts';
4+
import { dispatchCommand } from '../../../core/dispatch.ts';
5+
import { makeAndroidSession } from '../../../__tests__/test-utils/index.ts';
6+
import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts';
7+
8+
vi.mock('../../../core/dispatch.ts', async (importOriginal) => {
9+
const actual = await importOriginal<typeof import('../../../core/dispatch.ts')>();
10+
return {
11+
...actual,
12+
dispatchCommand: vi.fn(async () => ({})),
13+
};
14+
});
15+
16+
const mockDispatch = vi.mocked(dispatchCommand);
17+
const ANDROID_SCRIPT_ERROR = 'Unable to load script. Make sure you are running Metro.';
18+
19+
beforeEach(() => {
20+
mockDispatch.mockReset();
21+
mockDispatch.mockResolvedValue({});
22+
});
23+
24+
test('snapshot resolves @ref scope with the stored source after scoped output replaces refs', async () => {
25+
const sessionStore = makeSessionStore('agent-device-snapshot-scoped-refs-');
26+
const sessionName = 'android-ref-scope-repeat';
27+
const session = makeAndroidSession(sessionName, { snapshot: androidRefScopeSourceSnapshot() });
28+
sessionStore.set(sessionName, session);
29+
30+
mockDispatch.mockResolvedValue({
31+
nodes: scopedScriptErrorNodes(),
32+
truncated: false,
33+
backend: 'android',
34+
});
35+
36+
for (let attempt = 0; attempt < 2; attempt += 1) {
37+
const response = await requestScopedSnapshot(sessionName, sessionStore, '@e3');
38+
39+
expect(response?.ok).toBe(true);
40+
if (response?.ok) expect(response.data?.nodes).toHaveLength(2);
41+
}
42+
43+
expect(mockDispatch).toHaveBeenCalledTimes(2);
44+
expect(mockDispatch.mock.calls.map((call) => call[4])).toEqual([
45+
expect.objectContaining({ snapshotScope: ANDROID_SCRIPT_ERROR }),
46+
expect.objectContaining({ snapshotScope: ANDROID_SCRIPT_ERROR }),
47+
]);
48+
expect(sessionStore.get(sessionName)?.snapshot?.nodes).toHaveLength(2);
49+
expect(sessionStore.get(sessionName)?.snapshotScopeSource?.nodes[2]?.ref).toBe('e3');
50+
});
51+
52+
test('empty @ref-scoped snapshot output does not replace the stored session snapshot', async () => {
53+
const sessionStore = makeSessionStore('agent-device-snapshot-scoped-refs-');
54+
const sessionName = 'android-empty-scope-preserve';
55+
const session = makeAndroidSession(sessionName, { snapshot: currentScreenSnapshot() });
56+
sessionStore.set(sessionName, session);
57+
58+
mockDispatch.mockResolvedValue({
59+
nodes: [],
60+
truncated: false,
61+
backend: 'android',
62+
});
63+
64+
const response = await requestScopedSnapshot(sessionName, sessionStore, '@e1');
65+
66+
expect(response?.ok).toBe(true);
67+
if (response?.ok) expect(response.data?.nodes).toEqual([]);
68+
expect(sessionStore.get(sessionName)?.snapshot?.nodes[0]?.label).toBe('Current screen');
69+
});
70+
71+
function requestScopedSnapshot(
72+
sessionName: string,
73+
sessionStore: ReturnType<typeof makeSessionStore>,
74+
snapshotScope: string,
75+
) {
76+
return handleSnapshotCommands({
77+
req: {
78+
token: 't',
79+
session: sessionName,
80+
command: 'snapshot',
81+
positionals: [],
82+
flags: { snapshotScope },
83+
},
84+
sessionName,
85+
logPath: '/tmp/daemon.log',
86+
sessionStore,
87+
});
88+
}
89+
90+
function currentScreenSnapshot(): SnapshotState {
91+
return {
92+
nodes: [
93+
{ ref: 'e1', index: 0, depth: 0, type: 'android.widget.TextView', label: 'Current screen' },
94+
],
95+
createdAt: Date.now(),
96+
backend: 'android',
97+
};
98+
}
99+
100+
function androidRefScopeSourceSnapshot(): SnapshotState {
101+
return {
102+
nodes: [
103+
{
104+
ref: 'e1',
105+
index: 0,
106+
depth: 0,
107+
type: 'android.widget.FrameLayout',
108+
rect: { x: 0, y: 0, width: 390, height: 844 },
109+
},
110+
{
111+
ref: 'e2',
112+
index: 1,
113+
depth: 1,
114+
parentIndex: 0,
115+
type: 'androidx.recyclerview.widget.RecyclerView',
116+
rect: { x: 0, y: 80, width: 390, height: 600 },
117+
},
118+
{
119+
ref: 'e3',
120+
index: 2,
121+
depth: 2,
122+
parentIndex: 1,
123+
type: 'android.widget.TextView',
124+
label: ANDROID_SCRIPT_ERROR,
125+
value: ANDROID_SCRIPT_ERROR,
126+
rect: { x: 16, y: 120, width: 358, height: 200 },
127+
},
128+
{
129+
ref: 'e4',
130+
index: 3,
131+
depth: 3,
132+
parentIndex: 2,
133+
type: 'android.widget.TextView',
134+
label: 'loadJSBundleFromAssets',
135+
rect: { x: 16, y: 140, width: 358, height: 40 },
136+
},
137+
],
138+
createdAt: Date.now(),
139+
backend: 'android',
140+
};
141+
}
142+
143+
function scopedScriptErrorNodes(): RawSnapshotNode[] {
144+
return [
145+
{
146+
index: 0,
147+
depth: 0,
148+
type: 'android.widget.TextView',
149+
label: ANDROID_SCRIPT_ERROR,
150+
value: ANDROID_SCRIPT_ERROR,
151+
rect: { x: 16, y: 120, width: 358, height: 200 },
152+
},
153+
{
154+
index: 1,
155+
depth: 1,
156+
parentIndex: 0,
157+
type: 'android.widget.TextView',
158+
label: 'loadJSBundleFromAssets',
159+
rect: { x: 16, y: 140, width: 358, height: 40 },
160+
},
161+
];
162+
}

‎src/daemon/handlers/find.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { ensureDeviceReady } from '../device-ready.ts';
99
import { extractNodeText, findNearestHittableAncestor } from '../snapshot-processing.ts';
1010
import { readTextForNode } from './interaction-read.ts';
1111
import { captureSnapshot } from './snapshot-capture.ts';
12+
import { setSessionSnapshot } from '../session-snapshot.ts';
1213
import { errorResponse } from './response.ts';
1314
import { getActiveAndroidSnapshotFreshness } from '../android-snapshot-freshness.ts';
1415
import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts';
@@ -109,7 +110,7 @@ export async function handleFindCommands(params: {
109110
lastSnapshotAt = now;
110111
lastNodes = nodes;
111112
if (session) {
112-
session.snapshot = snapshot;
113+
setSessionSnapshot(session, snapshot);
113114
sessionStore.set(sessionName, session);
114115
}
115116
return { nodes, truncated: snapshot.truncated, backend: snapshot.backend };

‎src/daemon/handlers/interaction-runtime.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { createAgentDevice, localCommandPolicy } from '../../runtime.ts';
88
import { AppError } from '../../utils/errors.ts';
99
import type { SessionState } from '../types.ts';
1010
import { createUnsupportedArtifactAdapter } from '../runtime-artifacts.ts';
11+
import { setSessionSnapshot } from '../session-snapshot.ts';
1112
import type { InteractionHandlerParams } from './interaction-common.ts';
1213
import type { CaptureSnapshotForSession } from './interaction-snapshot.ts';
1314

@@ -34,7 +35,7 @@ export function createInteractionRuntime(
3435
: undefined,
3536
set: (record) => {
3637
if (!record.snapshot) return;
37-
session.snapshot = record.snapshot;
38+
setSessionSnapshot(session, record.snapshot);
3839
params.sessionStore.set(params.sessionName, session);
3940
},
4041
},

‎src/daemon/handlers/interaction-snapshot.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { SessionState } from '../types.ts';
44
import type { SnapshotState } from '../../utils/snapshot.ts';
55
import type { ContextFromFlags } from './interaction-common.ts';
66
import { captureSnapshot } from './snapshot-capture.ts';
7+
import { setSessionSnapshot } from '../session-snapshot.ts';
78

89
export type CaptureSnapshotForSession = (
910
session: SessionState,
@@ -38,7 +39,7 @@ export async function captureSnapshotForSession(
3839
logPath: dispatchContext.logPath ?? '',
3940
androidFreshnessMode: options.androidFreshnessMode,
4041
});
41-
session.snapshot = snapshot;
42+
setSessionSnapshot(session, snapshot);
4243
sessionStore.set(session.name, session);
43-
return session.snapshot;
44+
return snapshot;
4445
}

‎src/daemon/handlers/session-replay-heal.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { dispatchCommand } from '../../core/dispatch.ts';
2+
import { setSessionSnapshot } from '../session-snapshot.ts';
23
import {
34
attachRefs,
45
type RawSnapshotNode,
@@ -198,7 +199,7 @@ async function captureSnapshotForReplay(
198199
createdAt: Date.now(),
199200
backend: data?.backend,
200201
};
201-
session.snapshot = snapshot;
202+
setSessionSnapshot(session, snapshot);
202203
sessionStore.set(session.name, session);
203204
return snapshot;
204205
}

‎src/daemon/handlers/snapshot-capture.ts‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ export async function captureSnapshot(params: CaptureSnapshotParams): Promise<{
6060
const data = await captureSnapshotData(params);
6161
clearAndroidSnapshotFreshness(params.session);
6262
return {
63-
snapshot: buildSnapshotState(data, params.flags),
63+
snapshot: buildSnapshotState(data, resolveSnapshotStateFlags(params)),
6464
analysis: data.analysis,
6565
};
6666
}
@@ -146,7 +146,19 @@ async function captureSnapshotAttempt(
146146
const data = await captureSnapshotData(params);
147147
return {
148148
data,
149-
snapshot: buildSnapshotState(data, params.flags),
149+
snapshot: buildSnapshotState(data, resolveSnapshotStateFlags(params)),
150+
};
151+
}
152+
153+
function resolveSnapshotStateFlags(
154+
params: Pick<CaptureSnapshotParams, 'flags' | 'snapshotScope'>,
155+
): CaptureSnapshotParams['flags'] {
156+
if (params.snapshotScope === undefined) {
157+
return params.flags;
158+
}
159+
return {
160+
...params.flags,
161+
snapshotScope: params.snapshotScope,
150162
};
151163
}
152164

@@ -350,8 +362,16 @@ export function resolveSnapshotScope(
350362
if (!ref) {
351363
return errorResponse('INVALID_ARGS', `Invalid ref scope: ${snapshotScope}`);
352364
}
353-
const node = findNodeByRef(session.snapshot.nodes, ref);
354-
const resolved = node ? resolveRefLabel(node, session.snapshot.nodes) : undefined;
365+
const candidates = [
366+
session.snapshot,
367+
...(session.snapshotScopeSource ? [session.snapshotScopeSource] : []),
368+
];
369+
let resolved: string | undefined;
370+
for (const snapshot of candidates) {
371+
const node = findNodeByRef(snapshot.nodes, ref);
372+
resolved = node ? resolveRefLabel(node, snapshot.nodes) : undefined;
373+
if (resolved) break;
374+
}
355375
if (!resolved) {
356376
return errorResponse('COMMAND_FAILED', `Ref ${snapshotScope} not found or has no label`);
357377
}

‎src/daemon/request-router.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { handleInteractionCommands } from './handlers/interaction.ts';
2323
import { handleLeaseCommands } from './handlers/lease.ts';
2424
import { buildSnapshotState, captureSnapshotData } from './handlers/snapshot-capture.ts';
2525
import { assertSessionSelectorMatches } from './session-selector.ts';
26+
import { setSessionSnapshot } from './session-snapshot.ts';
2627
import { applyRequestLockPolicy } from './request-lock-policy.ts';
2728
import { resolveEffectiveSessionName } from './session-routing.ts';
2829
import { normalizeTenantId, resolveSessionIsolationMode } from './config.ts';
@@ -473,7 +474,7 @@ async function applyScreenshotOverlay(
473474
snapshotScope: undefined,
474475
});
475476
const overlaySnapshot = buildSnapshotState(overlaySnapshotData, undefined);
476-
session.snapshot = overlaySnapshot;
477+
setSessionSnapshot(session, overlaySnapshot);
477478
const overlayRefs = await annotateScreenshotWithRefs({
478479
screenshotPath: data.path as string,
479480
snapshot: overlaySnapshot,

0 commit comments

Comments
 (0)