diff --git a/src/daemon/handlers/__tests__/replay-heal.test.ts b/src/daemon/handlers/__tests__/replay-heal.test.ts index 4067d38dd..2b8512f33 100644 --- a/src/daemon/handlers/__tests__/replay-heal.test.ts +++ b/src/daemon/handlers/__tests__/replay-heal.test.ts @@ -365,6 +365,87 @@ test('replay --update heals selector in is command', async () => { assert.ok(rewrittenSelector.includes('auth_continue')); }); +test('replay --update heals numeric get text drift when numeric candidate value is unique', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-get-numeric-')); + const sessionsDir = path.join(tempRoot, 'sessions'); + const replayPath = path.join(tempRoot, 'replay.ad'); + const sessionStore = new SessionStore(sessionsDir); + const sessionName = 'heal-get-numeric-session'; + sessionStore.set(sessionName, makeSession(sessionName)); + + writeReplayFile(replayPath, { + ts: Date.now(), + command: 'get', + positionals: ['text', 'role="statictext" label="2" || label="2"'], + flags: {}, + result: {}, + }); + + const invokeCalls: string[] = []; + const invoke = async (request: DaemonRequest): Promise => { + if (request.command !== 'get') { + return { ok: false, error: { code: 'INVALID_ARGS', message: `unexpected command ${request.command}` } }; + } + const selector = request.positionals?.[1] ?? ''; + invokeCalls.push(selector); + if (selector.includes('label="2"')) { + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'selector stale' } }; + } + if (selector.includes('label="20"')) { + return { ok: true, data: { text: '20' } }; + } + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'unexpected selector' } }; + }; + + const dispatch = async (): Promise | void> => { + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeStaticText', + label: '20', + rect: { x: 0, y: 100, width: 100, height: 24 }, + enabled: true, + hittable: true, + }, + { + index: 1, + type: 'XCUIElementTypeStaticText', + label: 'Version: 0.84.0', + rect: { x: 0, y: 200, width: 220, height: 17 }, + enabled: true, + hittable: true, + }, + ], + truncated: false, + backend: 'xctest', + }; + }; + + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'replay', + positionals: [replayPath], + flags: { replayUpdate: true }, + }, + sessionName, + logPath: path.join(tempRoot, 'daemon.log'), + sessionStore, + invoke, + dispatch, + }); + + assert.ok(response); + assert.equal(response.ok, true, JSON.stringify(response)); + if (response.ok) { + assert.equal(response.data?.healed, 1); + assert.equal(response.data?.replayed, 1); + } + assert.equal(invokeCalls.length, 2); +}); + test('replay rejects legacy JSON payload files', async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-json-rejected-')); const sessionsDir = path.join(tempRoot, 'sessions'); diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index 74a1d8d5c..95507e77b 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -310,6 +310,7 @@ export async function handleInteractionCommands(params: { platform: session.device.platform, requireRect: false, requireUnique: true, + disambiguateAmbiguous: sub === 'text', }); if (!resolved) { return { diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 89de169ca..baf000a47 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -10,7 +10,7 @@ import { ensureDeviceReady } from '../device-ready.ts'; import { resolveIosAppStateFromSnapshots } from '../app-state.ts'; import { stopIosRunnerSession } from '../../platforms/ios/runner-client.ts'; import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; -import { pruneGroupNodes } from '../snapshot-processing.ts'; +import { extractNodeText, normalizeType, pruneGroupNodes } from '../snapshot-processing.ts'; import { buildSelectorChainForNode, resolveSelectorChain, @@ -517,6 +517,10 @@ async function healReplayAction(params: { const session = sessionStore.get(sessionName); if (!session) return null; const requiresRect = action.command === 'click' || action.command === 'fill'; + const allowDisambiguation = + action.command === 'click' || + action.command === 'fill' || + (action.command === 'get' && action.positionals?.[0] === 'text'); const snapshot = await captureSnapshotForReplay(session, action, logPath, requiresRect, dispatch, sessionStore); const selectorCandidates = collectReplaySelectorCandidates(action); for (const candidate of selectorCandidates) { @@ -526,7 +530,7 @@ async function healReplayAction(params: { platform: session.device.platform, requireRect: requiresRect, requireUnique: true, - disambiguateAmbiguous: requiresRect, + disambiguateAmbiguous: allowDisambiguation, }); if (!resolved) continue; const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, { @@ -580,6 +584,10 @@ async function healReplayAction(params: { }; } } + const numericDriftHeal = healNumericGetTextDrift(action, snapshot, session); + if (numericDriftHeal) { + return numericDriftHeal; + } return null; } @@ -697,6 +705,56 @@ function parseSelectorWaitPositionals(positionals: string[]): { }; } +function healNumericGetTextDrift( + action: SessionAction, + snapshot: SnapshotState, + session: SessionState, +): SessionAction | null { + if (action.command !== 'get') return null; + if (action.positionals?.[0] !== 'text') return null; + const selectorExpression = action.positionals?.[1]; + if (!selectorExpression) return null; + const chain = tryParseSelectorChain(selectorExpression); + if (!chain) return null; + + const roleFilters = new Set(); + let hasNumericTerm = false; + for (const selector of chain.selectors) { + for (const term of selector.terms) { + if (term.key === 'role' && typeof term.value === 'string') { + roleFilters.add(normalizeType(term.value)); + } + if ( + (term.key === 'text' || term.key === 'label' || term.key === 'value') && + typeof term.value === 'string' && + /^\d+$/.test(term.value.trim()) + ) { + hasNumericTerm = true; + } + } + } + if (!hasNumericTerm) return null; + + const numericNodes = snapshot.nodes.filter((node) => { + const text = extractNodeText(node).trim(); + if (!/^\d+$/.test(text)) return false; + if (roleFilters.size === 0) return true; + return roleFilters.has(normalizeType(node.type ?? '')); + }); + if (numericNodes.length === 0) return null; + const numericValues = uniqueStrings(numericNodes.map((node) => extractNodeText(node).trim())); + if (numericValues.length !== 1) return null; + + const targetNode = numericNodes[0]; + if (!targetNode) return null; + const selectorChain = buildSelectorChainForNode(targetNode, session.device.platform, { action: 'get' }); + if (selectorChain.length === 0) return null; + return { + ...action, + positionals: ['text', selectorChain.join(' || ')], + }; +} + function parseReplayScript(script: string): SessionAction[] { const actions: SessionAction[] = []; const lines = script.split(/\r?\n/); diff --git a/src/daemon/selectors.ts b/src/daemon/selectors.ts index 1540fede4..2db1fc626 100644 --- a/src/daemon/selectors.ts +++ b/src/daemon/selectors.ts @@ -483,7 +483,6 @@ function analyzeSelectorMatches( } if (!best) { best = node; - tie = false; continue; } const comparison = compareDisambiguationCandidates(node, best);