Skip to content

Commit 0bd3c01

Browse files
committed
fix: preserve ranked Maestro replay suggestions
1 parent 4cae9c2 commit 0bd3c01

2 files changed

Lines changed: 269 additions & 14 deletions

File tree

src/daemon/handlers/__tests__/session-replay-maestro-failure.test.ts

Lines changed: 192 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,18 @@ vi.mock('../../../core/dispatch.ts', async (importOriginal) => {
77
import fs from 'node:fs';
88
import os from 'node:os';
99
import path from 'node:path';
10-
import { buildTypedMaestroFailureReportProjection } from '../session-replay-maestro-failure.ts';
10+
import {
11+
buildTypedMaestroFailureReportProjection,
12+
buildTypedMaestroFailureResponse,
13+
} from '../session-replay-maestro-failure.ts';
1114
import { runReplayScriptFile } from '../session-replay-runtime.ts';
1215
import { SessionStore } from '../../session-store.ts';
1316
import { dispatchCommand } from '../../../core/dispatch.ts';
1417
import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts';
1518
import { baseReplayRequest as baseReq } from './session-replay-runtime.fixtures.ts';
19+
import type { MaestroCommand } from '../../../compat/maestro/program-ir.ts';
20+
import type { MaestroReplayPlan } from '../../../compat/maestro/replay-plan-types.ts';
21+
import type { SnapshotNode } from '../../../kernel/snapshot.ts';
1622

1723
const mockDispatchCommand = vi.mocked(dispatchCommand);
1824

@@ -21,6 +27,54 @@ beforeEach(() => {
2127
mockDispatchCommand.mockResolvedValue({});
2228
});
2329

30+
function makeMaestroPlan(): MaestroReplayPlan {
31+
return {
32+
kind: 'maestroReplayPlan',
33+
platform: 'ios',
34+
initialStaticEnv: {},
35+
steps: [],
36+
total: 1,
37+
digest: 'typed-maestro-test-plan',
38+
compatibility: {
39+
staticallyExecutedControls: 0,
40+
staticallySkippedControls: 0,
41+
},
42+
};
43+
}
44+
45+
async function buildFailureResponse(
46+
command: MaestroCommand,
47+
nodes: SnapshotNode[],
48+
): Promise<Extract<Awaited<ReturnType<typeof buildTypedMaestroFailureResponse>>, { ok: false }>> {
49+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-suggestions-'));
50+
const sessionName = 'default';
51+
const sessionStore = new SessionStore(path.join(root, 'sessions'));
52+
sessionStore.set(sessionName, makeIosSession(sessionName));
53+
mockDispatchCommand.mockResolvedValue({ nodes, truncated: false, backend: 'xctest' });
54+
const response = await buildTypedMaestroFailureResponse({
55+
error: { code: 'COMMAND_FAILED', message: 'typed Maestro action failed' },
56+
event: {
57+
command,
58+
source: command.source,
59+
generation: 0,
60+
stepIndex: 1,
61+
stepTotal: 1,
62+
durationMs: 12,
63+
error: new Error('typed Maestro action failed'),
64+
artifactPaths: [],
65+
expandedVariables: {},
66+
},
67+
plan: makeMaestroPlan(),
68+
replayPath: path.join(root, 'flow.yaml'),
69+
req: baseReq({ flags: { replayBackend: 'maestro', platform: 'ios' } }),
70+
sessionName,
71+
sessionStore,
72+
logPath: path.join(root, 'daemon.log'),
73+
});
74+
if (response.ok) throw new Error('expected typed Maestro failure response');
75+
return response;
76+
}
77+
2478
test('typed Maestro failure projection is report-only and preserves authored provenance', () => {
2579
const command = {
2680
kind: 'tapOn' as const,
@@ -221,3 +275,140 @@ test('typed Maestro nested scopes scrub failure values after unwind and keep ret
221275
}),
222276
]);
223277
});
278+
279+
test('typed Maestro suggestions rank visible childOf candidates and exclude out-of-scope nodes', async () => {
280+
const command = {
281+
kind: 'tapOn' as const,
282+
source: { path: '/flows/actions.yaml', line: 4 },
283+
target: { space: 'target' as const, selector: { text: 'save.*' } },
284+
childOf: { id: 'actions' },
285+
} satisfies Extract<MaestroCommand, { kind: 'tapOn' }>;
286+
const response = await buildFailureResponse(command, [
287+
{
288+
ref: 'e1',
289+
index: 0,
290+
type: 'Application',
291+
rect: { x: 0, y: 0, width: 402, height: 874 },
292+
},
293+
{
294+
ref: 'e2',
295+
index: 1,
296+
parentIndex: 0,
297+
identifier: 'actions',
298+
type: 'View',
299+
rect: { x: 0, y: 0, width: 402, height: 300 },
300+
},
301+
{
302+
ref: 'e3',
303+
index: 2,
304+
parentIndex: 1,
305+
identifier: 'save-primary',
306+
label: 'Primary',
307+
type: 'Button',
308+
rect: { x: 16, y: 40, width: 140, height: 44 },
309+
hittable: true,
310+
},
311+
{
312+
ref: 'e4',
313+
index: 3,
314+
parentIndex: 1,
315+
identifier: 'secondary',
316+
label: 'Save secondary',
317+
type: 'Button',
318+
rect: { x: 16, y: 96, width: 140, height: 44 },
319+
hittable: true,
320+
},
321+
{
322+
ref: 'e5',
323+
index: 4,
324+
parentIndex: 0,
325+
identifier: 'save-outside',
326+
label: 'Save outside',
327+
type: 'Button',
328+
rect: { x: 16, y: 152, width: 140, height: 44 },
329+
hittable: true,
330+
},
331+
{
332+
ref: 'e6',
333+
index: 5,
334+
parentIndex: 1,
335+
identifier: 'save-hidden',
336+
label: 'Save hidden',
337+
type: 'Button',
338+
rect: { x: 16, y: 208, width: 140, height: 0 },
339+
hittable: false,
340+
},
341+
]);
342+
const divergence = response.error.details?.divergence as {
343+
suggestionCount: number;
344+
suggestions: Array<{ selector: string; basis: string }>;
345+
};
346+
347+
expect(divergence.suggestions).toHaveLength(2);
348+
expect(divergence.suggestions.map(({ basis }) => basis)).toEqual(['id', 'label']);
349+
expect(divergence.suggestions[0]?.selector).toContain('save-primary');
350+
expect(divergence.suggestions[1]?.selector).toContain('Save secondary');
351+
expect(divergence.suggestionCount).toBe(2);
352+
});
353+
354+
test('typed Maestro text matching an identifier reports id basis', async () => {
355+
const command = {
356+
kind: 'tapOn' as const,
357+
source: { path: '/flows/actions.yaml', line: 4 },
358+
target: { space: 'target' as const, selector: { text: 'accessibility-save' } },
359+
} satisfies Extract<MaestroCommand, { kind: 'tapOn' }>;
360+
const response = await buildFailureResponse(command, [
361+
{
362+
ref: 'e1',
363+
index: 0,
364+
type: 'Application',
365+
rect: { x: 0, y: 0, width: 402, height: 874 },
366+
},
367+
{
368+
ref: 'e2',
369+
index: 1,
370+
parentIndex: 0,
371+
identifier: 'accessibility-save',
372+
type: 'Button',
373+
rect: { x: 16, y: 40, width: 140, height: 44 },
374+
hittable: true,
375+
},
376+
]);
377+
const divergence = response.error.details?.divergence as {
378+
suggestions: Array<{ basis: string }>;
379+
};
380+
381+
expect(divergence.suggestions).toEqual([expect.objectContaining({ basis: 'id' })]);
382+
});
383+
384+
test('typed Maestro suggestions retain total count before the five-entry cap', async () => {
385+
const command = {
386+
kind: 'tapOn' as const,
387+
source: { path: '/flows/actions.yaml', line: 4 },
388+
target: { space: 'target' as const, selector: { label: 'Save' } },
389+
} satisfies Extract<MaestroCommand, { kind: 'tapOn' }>;
390+
const response = await buildFailureResponse(command, [
391+
{
392+
ref: 'e1',
393+
index: 0,
394+
type: 'Application',
395+
rect: { x: 0, y: 0, width: 402, height: 874 },
396+
},
397+
...Array.from({ length: 6 }, (_, offset) => ({
398+
ref: `e${offset + 2}`,
399+
index: offset + 1,
400+
parentIndex: 0,
401+
label: 'Save',
402+
type: 'Button',
403+
rect: { x: 16, y: 40 + offset * 50, width: 140, height: 44 },
404+
hittable: true,
405+
})),
406+
]);
407+
const divergence = response.error.details?.divergence as {
408+
suggestionCount: number;
409+
suggestions: unknown[];
410+
};
411+
412+
expect(divergence.suggestionCount).toBe(6);
413+
expect(divergence.suggestions).toHaveLength(5);
414+
});

src/daemon/handlers/session-replay-maestro-failure.ts

Lines changed: 77 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,18 @@ import { formatMaestroCommandProgress } from '../../compat/maestro/progress.ts';
33
import type { MaestroCommand, MaestroSelector } from '../../compat/maestro/program-ir.ts';
44
import { evaluateMaestroReplayResume } from '../../compat/maestro/replay-plan.ts';
55
import type { MaestroReplayPlan } from '../../compat/maestro/replay-plan-types.ts';
6-
import { resolveMaestroTargetFromSnapshot } from '../../compat/maestro/runtime-targets.ts';
6+
import { findMaestroTypedSelectorMatches } from '../../compat/maestro/runtime-target-matching.ts';
7+
import {
8+
filterVisibleMaestroMatches,
9+
matchesMaestroTypedSelector,
10+
} from '../../compat/maestro/runtime-target-policy.ts';
11+
import { normalizeMaestroSnapshotMatches } from '../../compat/maestro/runtime-target-ranking.ts';
712
import type { DaemonError } from '../../kernel/contracts.ts';
813
import type { SnapshotNode } from '../../kernel/snapshot.ts';
14+
import {
15+
buildSnapshotNodeByIndex,
16+
isDescendantOfSnapshotNode,
17+
} from '../../snapshot/snapshot-processing.ts';
918
import {
1019
REPLAY_DIVERGENCE_SUGGESTION_LIMIT,
1120
createReplayDivergenceSanitizer,
@@ -192,17 +201,57 @@ function collectTypedMaestroSuggestions(params: {
192201
const query = typedSuggestionQuery(params.command);
193202
if (!query || (params.platform !== 'android' && params.platform !== 'ios')) return [];
194203
const snapshot = { createdAt: Date.now(), nodes: params.nodes };
195-
const resolution = resolveMaestroTargetFromSnapshot(snapshot, query, params.platform);
196-
if (!resolution.ok) return [];
197-
return [
198-
buildReplayDivergenceSuggestionForNode({
199-
node: resolution.node,
200-
session: params.session,
201-
action: params.action,
202-
basis: suggestionBasis(query.selector),
203-
sanitize: params.sanitize,
204-
}),
205-
];
204+
const candidates = collectTypedMaestroCandidates(snapshot, query, params.platform);
205+
const byNode = new Map<number, { node: SnapshotNode; basis: TypedSuggestionBasis }>();
206+
for (const node of candidates) {
207+
const basis = suggestionBasis(query.selector, node);
208+
const existing = byNode.get(node.index);
209+
if (!existing || typedSuggestionBasisRank(basis) < typedSuggestionBasisRank(existing.basis)) {
210+
byNode.set(node.index, { node, basis });
211+
}
212+
}
213+
return [...byNode.values()]
214+
.sort(
215+
(left, right) =>
216+
typedSuggestionBasisRank(left.basis) - typedSuggestionBasisRank(right.basis) ||
217+
left.node.index - right.node.index,
218+
)
219+
.map(({ node, basis }) =>
220+
buildReplayDivergenceSuggestionForNode({
221+
node,
222+
session: params.session,
223+
action: params.action,
224+
basis,
225+
sanitize: params.sanitize,
226+
}),
227+
);
228+
}
229+
230+
type TypedSuggestionBasis = 'id' | 'label' | 'other';
231+
232+
function collectTypedMaestroCandidates(
233+
snapshot: { nodes: SnapshotNode[]; createdAt: number },
234+
query: TypedSuggestionQuery,
235+
platform: Extract<MaestroReplayPlan['platform'], 'android' | 'ios'>,
236+
): SnapshotNode[] {
237+
let matches = findMaestroTypedSelectorMatches(snapshot, query.selector);
238+
if (query.childOf) {
239+
const parents = findMaestroTypedSelectorMatches(snapshot, query.childOf);
240+
if (parents.length === 0) return [];
241+
const nodeByIndex = buildSnapshotNodeByIndex(snapshot.nodes);
242+
matches = matches.filter((node) =>
243+
parents.some((parent) =>
244+
isDescendantOfSnapshotNode(snapshot.nodes, node, parent, nodeByIndex),
245+
),
246+
);
247+
}
248+
249+
const visible = filterVisibleMaestroMatches({
250+
nodes: snapshot.nodes,
251+
matches,
252+
platform,
253+
});
254+
return normalizeMaestroSnapshotMatches(snapshot.nodes, visible.matches, query.selector, platform);
206255
}
207256

208257
type TypedSuggestionQuery = {
@@ -275,12 +324,27 @@ function observationSuggestion(command: ObservationCommand): TypedSuggestionQuer
275324
}
276325
}
277326

278-
function suggestionBasis(selector: MaestroSelector): 'id' | 'label' | 'other' {
327+
function suggestionBasis(selector: MaestroSelector, node: SnapshotNode): TypedSuggestionBasis {
279328
if (selector.id !== undefined) return 'id';
329+
if (
330+
selector.text !== undefined &&
331+
matchesMaestroTypedSelector(
332+
{ ...node, label: undefined, value: undefined },
333+
{ text: selector.text },
334+
)
335+
) {
336+
return 'id';
337+
}
280338
if (selector.text !== undefined || selector.label !== undefined) return 'label';
281339
return 'other';
282340
}
283341

342+
function typedSuggestionBasisRank(basis: TypedSuggestionBasis): number {
343+
if (basis === 'id') return 0;
344+
if (basis === 'label') return 2;
345+
return 3;
346+
}
347+
284348
function reportCommandForCapture(command: string): string {
285349
if (command === 'tapOn' || command === 'doubleTapOn') return 'click';
286350
if (command === 'longPressOn') return 'longpress';

0 commit comments

Comments
 (0)