From 215ad3349f6408c318c4c6f7f1d054fd6e52b980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Feb 2026 12:51:13 +0100 Subject: [PATCH 1/4] Refine selector resolution and simplify replay healing --- src/daemon/__tests__/selectors.test.ts | 87 +++++++++++ .../handlers/__tests__/replay-heal.test.ts | 59 ++++++++ src/daemon/handlers/interaction.ts | 6 +- src/daemon/handlers/session.ts | 15 +- src/daemon/selectors.ts | 142 +++++++++++++++--- src/daemon/snapshot-processing.ts | 10 +- 6 files changed, 292 insertions(+), 27 deletions(-) diff --git a/src/daemon/__tests__/selectors.test.ts b/src/daemon/__tests__/selectors.test.ts index 07862303d..11ba3be42 100644 --- a/src/daemon/__tests__/selectors.test.ts +++ b/src/daemon/__tests__/selectors.test.ts @@ -74,6 +74,51 @@ test('resolveSelectorChain falls back when first selector is ambiguous', () => { assert.equal(resolved.node.ref, 'e2'); }); +test('resolveSelectorChain keeps strict ambiguity behavior by default', () => { + const chain = parseSelectorChain('label="Continue"'); + const resolved = resolveSelectorChain(nodes, chain, { + platform: 'ios', + requireRect: true, + requireUnique: true, + }); + assert.equal(resolved, null); +}); + +test('resolveSelectorChain disambiguates to deeper/smaller matching node when enabled', () => { + const disambiguationNodes: SnapshotState['nodes'] = [ + { + ref: 'e1', + index: 0, + type: 'Other', + label: 'Press me', + rect: { x: 0, y: 0, width: 300, height: 300 }, + depth: 1, + enabled: true, + hittable: true, + }, + { + ref: 'e2', + index: 1, + type: 'Other', + label: 'Press me', + rect: { x: 10, y: 10, width: 100, height: 20 }, + depth: 2, + enabled: true, + hittable: true, + }, + ]; + const chain = parseSelectorChain('role="other" label="Press me" || label="Press me"'); + const resolved = resolveSelectorChain(disambiguationNodes, chain, { + platform: 'ios', + requireRect: true, + requireUnique: true, + disambiguateAmbiguous: true, + }); + assert.ok(resolved); + assert.equal(resolved.node.ref, 'e2'); + assert.equal(resolved.matches, 2); +}); + test('findSelectorChainMatch returns first matching selector for existence checks', () => { const chain = parseSelectorChain('label="Continue" || id=auth_continue'); const match = findSelectorChainMatch(nodes, chain, { @@ -91,12 +136,31 @@ test('splitSelectorFromArgs extracts selector prefix and trailing value', () => assert.deepEqual(split.rest, ['qa@example.com']); }); +test('splitSelectorFromArgs prefers trailing token for value when requested', () => { + const split = splitSelectorFromArgs(['label="Filter"', 'visible=true'], { preferTrailingValue: true }); + assert.ok(split); + assert.equal(split.selectorExpression, 'label="Filter"'); + assert.deepEqual(split.rest, ['visible=true']); +}); + +test('splitSelectorFromArgs keeps full selector when trailing value preference is disabled', () => { + const split = splitSelectorFromArgs(['label="Filter"', 'visible=true']); + assert.ok(split); + assert.equal(split.selectorExpression, 'label="Filter" visible=true'); + assert.deepEqual(split.rest, []); +}); + test('parseSelectorChain rejects unknown keys and malformed quotes', () => { assert.throws(() => parseSelectorChain('foo=bar'), /Unknown selector key/i); assert.throws(() => parseSelectorChain('label="unclosed'), /Unclosed quote/i); assert.throws(() => parseSelectorChain(''), /cannot be empty/i); }); +test('parseSelectorChain handles quoted values ending in escaped backslashes', () => { + const chain = parseSelectorChain('label="path\\\\" || id=auth_continue'); + assert.equal(chain.selectors.length, 2); +}); + test('isSelectorToken only accepts known keys for key=value tokens', () => { assert.equal(isSelectorToken('id=foo'), true); assert.equal(isSelectorToken('editable=true'), true); @@ -126,3 +190,26 @@ test('buildSelectorChainForNode prefers id and adds editable for fill action', ( assert.ok(chain.some((entry) => entry.includes('id='))); assert.ok(chain.some((entry) => entry.includes('editable=true'))); }); + +test('role selector normalization matches Android class names by leaf type', () => { + const androidNodes: SnapshotState['nodes'] = [ + { + ref: 'a1', + index: 0, + type: 'android.widget.Button', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 0, y: 0, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ]; + const chain = parseSelectorChain('role=button label="Continue"'); + const resolved = resolveSelectorChain(androidNodes, chain, { + platform: 'android', + requireRect: true, + requireUnique: true, + }); + assert.ok(resolved); + assert.equal(resolved.node.ref, 'a1'); +}); diff --git a/src/daemon/handlers/__tests__/replay-heal.test.ts b/src/daemon/handlers/__tests__/replay-heal.test.ts index 5cea1fa65..4067d38dd 100644 --- a/src/daemon/handlers/__tests__/replay-heal.test.ts +++ b/src/daemon/handlers/__tests__/replay-heal.test.ts @@ -234,6 +234,65 @@ test('replay without --update does not heal or rewrite', async () => { assert.equal(fs.readFileSync(replayPath, 'utf8'), originalPayload); }); +test('replay --update skips malformed selector candidates and preserves replay error context', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-malformed-candidate-')); + const sessionsDir = path.join(tempRoot, 'sessions'); + const replayPath = path.join(tempRoot, 'replay.ad'); + const sessionStore = new SessionStore(sessionsDir); + const sessionName = 'malformed-candidate-session'; + sessionStore.set(sessionName, makeSession(sessionName)); + + writeReplayFile(replayPath, { + ts: Date.now(), + command: 'click', + positionals: ['id="old_continue" ||'], + flags: {}, + result: {}, + }); + + const dispatch = async (): Promise | void> => { + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 10, width: 100, height: 44 }, + enabled: true, + hittable: true, + }, + ], + truncated: false, + backend: 'xctest', + }; + }; + + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'replay', + positionals: [replayPath], + flags: { replayUpdate: true }, + }, + sessionName, + logPath: path.join(tempRoot, 'daemon.log'), + sessionStore, + invoke: async () => ({ ok: false, error: { code: 'COMMAND_FAILED', message: 'selector stale' } }), + dispatch, + }); + + assert.ok(response); + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'COMMAND_FAILED'); + assert.match(response.error.message, /Replay failed at step 1/); + assert.equal(response.error.details?.step, 1); + assert.equal(response.error.details?.action, 'click'); + } +}); + test('replay --update heals selector in is command', async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-is-')); const sessionsDir = path.join(tempRoot, 'sessions'); diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index a518844b9..006f155e1 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -90,6 +90,7 @@ export async function handleInteractionCommands(params: { platform: session.device.platform, requireRect: true, requireUnique: true, + disambiguateAmbiguous: true, }); if (!resolved || !resolved.node.rect) { return { @@ -180,7 +181,7 @@ export async function handleInteractionCommands(params: { error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, }; } - const selectorArgs = splitSelectorFromArgs(req.positionals ?? []); + const selectorArgs = splitSelectorFromArgs(req.positionals ?? [], { preferTrailingValue: true }); if (selectorArgs) { if (selectorArgs.rest.length === 0) { return { ok: false, error: { code: 'INVALID_ARGS', message: 'fill requires text after selector' } }; @@ -197,6 +198,7 @@ export async function handleInteractionCommands(params: { platform: session.device.platform, requireRect: true, requireUnique: true, + disambiguateAmbiguous: true, }); if (!resolved || !resolved.node.rect) { return { @@ -368,7 +370,7 @@ export async function handleInteractionCommands(params: { }; } const selectorArgs = req.positionals.slice(1); - const split = splitSelectorFromArgs(selectorArgs); + const split = splitSelectorFromArgs(selectorArgs, { preferTrailingValue: predicate === 'text' }); if (!split) { return { ok: false, diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 098a8e99b..111875738 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -11,7 +11,7 @@ import { resolveIosAppStateFromSnapshots } from '../app-state.ts'; import { stopIosRunnerSession } from '../../platforms/ios/runner-client.ts'; import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; import { pruneGroupNodes } from '../snapshot-processing.ts'; -import { buildSelectorChainForNode, parseSelectorChain, resolveSelectorChain, splitSelectorFromArgs } from '../selectors.ts'; +import { buildSelectorChainForNode, resolveSelectorChain, splitSelectorFromArgs, tryParseSelectorChain } from '../selectors.ts'; import { inferFillText, uniqueStrings } from '../action-utils.ts'; type ReinstallOps = { @@ -514,11 +514,13 @@ async function healReplayAction(params: { const snapshot = await captureSnapshotForReplay(session, action, logPath, requiresRect, dispatch, sessionStore); const selectorCandidates = collectReplaySelectorCandidates(action); for (const candidate of selectorCandidates) { - const chain = parseSelectorChain(candidate); + const chain = tryParseSelectorChain(candidate); + if (!chain) continue; const resolved = resolveSelectorChain(snapshot.nodes, chain, { platform: session.device.platform, requireRect: requiresRect, requireUnique: true, + disambiguateAmbiguous: action.command === 'click' || action.command === 'fill', }); if (!resolved) continue; const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, { @@ -550,7 +552,9 @@ async function healReplayAction(params: { if (action.command === 'is') { const predicate = action.positionals?.[0]; if (!predicate) continue; - const split = splitSelectorFromArgs(action.positionals.slice(1)); + const split = splitSelectorFromArgs(action.positionals.slice(1), { + preferTrailingValue: predicate === 'text', + }); const expectedText = split?.rest.join(' ').trim() ?? ''; const nextPositionals = [predicate, selectorExpression]; if (predicate === 'text' && expectedText.length > 0) { @@ -641,7 +645,10 @@ function collectReplaySelectorCandidates(action: SessionAction): string[] { } } if (action.command === 'is') { - const split = splitSelectorFromArgs(action.positionals.slice(1)); + const predicate = action.positionals?.[0]; + const split = splitSelectorFromArgs(action.positionals.slice(1), { + preferTrailingValue: predicate === 'text', + }); if (split) { result.push(split.selectorExpression); } diff --git a/src/daemon/selectors.ts b/src/daemon/selectors.ts index fa4b5ab03..abcdb90fe 100644 --- a/src/daemon/selectors.ts +++ b/src/daemon/selectors.ts @@ -85,25 +85,41 @@ export function resolveSelectorChain( platform: 'ios' | 'android'; requireRect?: boolean; requireUnique?: boolean; + disambiguateAmbiguous?: boolean; }, ): SelectorResolution | null { const requireRect = options.requireRect ?? false; const requireUnique = options.requireUnique ?? true; + const disambiguateAmbiguous = options.disambiguateAmbiguous ?? false; const diagnostics: SelectorDiagnostics[] = []; for (let i = 0; i < chain.selectors.length; i += 1) { const selector = chain.selectors[i]; - const matches = nodes.filter((node) => { - if (requireRect && !node.rect) return false; - return matchesSelector(node, selector, options.platform); + const summary = countSelectorMatches(nodes, selector, { + platform: options.platform, + requireRect, }); - diagnostics.push({ selector: selector.raw, matches: matches.length }); - if (matches.length === 0) continue; - if (requireUnique && matches.length !== 1) continue; + diagnostics.push({ selector: selector.raw, matches: summary.count }); + if (summary.count === 0 || !summary.firstNode) continue; + if (requireUnique && summary.count !== 1) { + if (!disambiguateAmbiguous) continue; + const disambiguatedNode = pickDisambiguatedNode(nodes, selector, { + platform: options.platform, + requireRect, + }); + if (!disambiguatedNode) continue; + return { + node: disambiguatedNode, + selector, + selectorIndex: i, + matches: summary.count, + diagnostics, + }; + } return { - node: matches[0], + node: summary.firstNode, selector, selectorIndex: i, - matches: matches.length, + matches: summary.count, diagnostics, }; } @@ -122,13 +138,13 @@ export function findSelectorChainMatch( const diagnostics: SelectorDiagnostics[] = []; for (let i = 0; i < chain.selectors.length; i += 1) { const selector = chain.selectors[i]; - const matches = nodes.filter((node) => { - if (requireRect && !node.rect) return false; - return matchesSelector(node, selector, options.platform); + const summary = countSelectorMatches(nodes, selector, { + platform: options.platform, + requireRect, }); - diagnostics.push({ selector: selector.raw, matches: matches.length }); - if (matches.length > 0) { - return { selectorIndex: i, selector, matches: matches.length, diagnostics }; + diagnostics.push({ selector: selector.raw, matches: summary.count }); + if (summary.count > 0) { + return { selectorIndex: i, selector, matches: summary.count, diagnostics }; } } return null; @@ -162,18 +178,35 @@ export function isSelectorToken(token: string): boolean { return ALL_KEYS.has(trimmed.toLowerCase() as SelectorKey); } -export function splitSelectorFromArgs(args: string[]): { selectorExpression: string; rest: string[] } | null { +export function splitSelectorFromArgs( + args: string[], + options: { preferTrailingValue?: boolean } = {}, +): { selectorExpression: string; rest: string[] } | null { if (args.length === 0) return null; + const preferTrailingValue = options.preferTrailingValue ?? false; let i = 0; + const boundaries: number[] = []; while (i < args.length && isSelectorToken(args[i])) { i += 1; + const candidate = args.slice(0, i).join(' ').trim(); + if (!candidate) continue; + if (tryParseSelectorChain(candidate)) { + boundaries.push(i); + } + } + if (boundaries.length === 0) return null; + let boundary = boundaries[boundaries.length - 1]; + if (preferTrailingValue) { + const boundaryWithRest = [...boundaries].reverse().find((index) => index < args.length); + if (boundaryWithRest !== undefined) { + boundary = boundaryWithRest; + } } - if (i === 0) return null; - const selectorExpression = args.slice(0, i).join(' ').trim(); + const selectorExpression = args.slice(0, boundary).join(' ').trim(); if (!selectorExpression) return null; return { selectorExpression, - rest: args.slice(i), + rest: args.slice(boundary), }; } @@ -318,7 +351,7 @@ function splitByFallback(expression: string): string[] { let quote: '"' | "'" | null = null; for (let i = 0; i < expression.length; i += 1) { const ch = expression[i]; - if ((ch === '"' || ch === "'") && expression[i - 1] !== '\\') { + if ((ch === '"' || ch === "'") && !isEscapedQuote(expression, i)) { if (!quote) { quote = ch; } else if (quote === ch) { @@ -353,7 +386,7 @@ function tokenize(segment: string): string[] { let quote: '"' | "'" | null = null; for (let i = 0; i < segment.length; i += 1) { const ch = segment[i]; - if ((ch === '"' || ch === "'") && segment[i - 1] !== '\\') { + if ((ch === '"' || ch === "'") && !isEscapedQuote(segment, i)) { if (!quote) { quote = ch; } else if (quote === ch) { @@ -421,3 +454,72 @@ function normalizeSelectorText(value: string | undefined): string | null { if (!trimmed) return null; return trimmed; } + +function countSelectorMatches( + nodes: SnapshotState['nodes'], + selector: Selector, + options: { platform: 'ios' | 'android'; requireRect: boolean }, +): { count: number; firstNode: SnapshotNode | null } { + let count = 0; + let firstNode: SnapshotNode | null = null; + for (const node of nodes) { + if (options.requireRect && !node.rect) continue; + if (!matchesSelector(node, selector, options.platform)) continue; + count += 1; + if (!firstNode) { + firstNode = node; + } + } + return { count, firstNode }; +} + +function pickDisambiguatedNode( + nodes: SnapshotState['nodes'], + selector: Selector, + options: { platform: 'ios' | 'android'; requireRect: boolean }, +): SnapshotNode | null { + let best: SnapshotNode | null = null; + let tie = false; + for (const node of nodes) { + if (options.requireRect && !node.rect) continue; + if (!matchesSelector(node, selector, options.platform)) continue; + if (!best) { + best = node; + tie = false; + continue; + } + const comparison = compareDisambiguationCandidates(node, best); + if (comparison > 0) { + best = node; + tie = false; + continue; + } + if (comparison === 0) { + tie = true; + } + } + return tie ? null : best; +} + +function compareDisambiguationCandidates(a: SnapshotNode, b: SnapshotNode): number { + const depthA = a.depth ?? 0; + const depthB = b.depth ?? 0; + if (depthA !== depthB) return depthA > depthB ? 1 : -1; + const areaA = areaOfNode(a); + const areaB = areaOfNode(b); + if (areaA !== areaB) return areaA < areaB ? 1 : -1; + return 0; +} + +function areaOfNode(node: SnapshotNode): number { + if (!node.rect) return Number.POSITIVE_INFINITY; + return node.rect.width * node.rect.height; +} + +function isEscapedQuote(source: string, index: number): boolean { + let backslashCount = 0; + for (let i = index - 1; i >= 0 && source[i] === '\\'; i -= 1) { + backslashCount += 1; + } + return backslashCount % 2 === 1; +} diff --git a/src/daemon/snapshot-processing.ts b/src/daemon/snapshot-processing.ts index e7f8b4a28..a755d21dc 100644 --- a/src/daemon/snapshot-processing.ts +++ b/src/daemon/snapshot-processing.ts @@ -78,10 +78,18 @@ export function pruneGroupNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { } export function normalizeType(type: string): string { - let value = type.replace(/XCUIElementType/gi, '').toLowerCase(); + let value = type.trim().replace(/XCUIElementType/gi, '').toLowerCase(); if (value.startsWith('ax')) { value = value.replace(/^ax/, ''); } + if (value.includes('.')) { + const suffix = value.split('.').pop(); + if (suffix) value = suffix; + } + if (value.includes('/')) { + const suffix = value.split('/').pop(); + if (suffix) value = suffix; + } return value; } From 45c13fd693dd1d8b320e81dcc10b99291eb4d783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Feb 2026 12:59:20 +0100 Subject: [PATCH 2/4] Address review feedback on selector internals --- src/daemon/__tests__/selectors.test.ts | 46 ++++++++++++++++ src/daemon/handlers/interaction.ts | 4 +- src/daemon/handlers/session.ts | 20 +++---- src/daemon/selectors.ts | 74 ++++++++++++++++---------- src/daemon/snapshot-processing.ts | 10 ++-- 5 files changed, 106 insertions(+), 48 deletions(-) diff --git a/src/daemon/__tests__/selectors.test.ts b/src/daemon/__tests__/selectors.test.ts index 11ba3be42..3d7cbbf99 100644 --- a/src/daemon/__tests__/selectors.test.ts +++ b/src/daemon/__tests__/selectors.test.ts @@ -119,6 +119,52 @@ test('resolveSelectorChain disambiguates to deeper/smaller matching node when en assert.equal(resolved.matches, 2); }); +test('resolveSelectorChain disambiguation tie falls back to next selector', () => { + const tieNodes: SnapshotState['nodes'] = [ + { + ref: 'e1', + index: 0, + type: 'Other', + label: 'Press me', + rect: { x: 0, y: 0, width: 100, height: 20 }, + depth: 2, + enabled: true, + hittable: true, + }, + { + ref: 'e2', + index: 1, + type: 'Other', + label: 'Press me', + rect: { x: 0, y: 40, width: 100, height: 20 }, + depth: 2, + enabled: true, + hittable: true, + }, + { + ref: 'e3', + index: 2, + type: 'Other', + label: 'Press me', + identifier: 'press_me_unique', + rect: { x: 0, y: 80, width: 100, height: 20 }, + depth: 2, + enabled: true, + hittable: true, + }, + ]; + const chain = parseSelectorChain('label="Press me" || id="press_me_unique"'); + const resolved = resolveSelectorChain(tieNodes, chain, { + platform: 'ios', + requireRect: true, + requireUnique: true, + disambiguateAmbiguous: true, + }); + assert.ok(resolved); + assert.equal(resolved.selectorIndex, 1); + assert.equal(resolved.node.ref, 'e3'); +}); + test('findSelectorChainMatch returns first matching selector for existence checks', () => { const chain = parseSelectorChain('label="Continue" || id=auth_continue'); const match = findSelectorChainMatch(nodes, chain, { diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index 006f155e1..74a1d8d5c 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -12,6 +12,7 @@ import { formatSelectorFailure, parseSelectorChain, resolveSelectorChain, + splitIsSelectorArgs, splitSelectorFromArgs, } from '../selectors.ts'; @@ -369,8 +370,7 @@ export async function handleInteractionCommands(params: { error: { code: 'UNSUPPORTED_OPERATION', message: 'is is not supported on this device' }, }; } - const selectorArgs = req.positionals.slice(1); - const split = splitSelectorFromArgs(selectorArgs, { preferTrailingValue: predicate === 'text' }); + const { split } = splitIsSelectorArgs(req.positionals); if (!split) { return { ok: false, diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 111875738..89de169ca 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -11,7 +11,13 @@ import { resolveIosAppStateFromSnapshots } from '../app-state.ts'; import { stopIosRunnerSession } from '../../platforms/ios/runner-client.ts'; import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; import { pruneGroupNodes } from '../snapshot-processing.ts'; -import { buildSelectorChainForNode, resolveSelectorChain, splitSelectorFromArgs, tryParseSelectorChain } from '../selectors.ts'; +import { + buildSelectorChainForNode, + resolveSelectorChain, + splitIsSelectorArgs, + splitSelectorFromArgs, + tryParseSelectorChain, +} from '../selectors.ts'; import { inferFillText, uniqueStrings } from '../action-utils.ts'; type ReinstallOps = { @@ -520,7 +526,7 @@ async function healReplayAction(params: { platform: session.device.platform, requireRect: requiresRect, requireUnique: true, - disambiguateAmbiguous: action.command === 'click' || action.command === 'fill', + disambiguateAmbiguous: requiresRect, }); if (!resolved) continue; const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, { @@ -550,11 +556,8 @@ async function healReplayAction(params: { }; } if (action.command === 'is') { - const predicate = action.positionals?.[0]; + const { predicate, split } = splitIsSelectorArgs(action.positionals); if (!predicate) continue; - const split = splitSelectorFromArgs(action.positionals.slice(1), { - preferTrailingValue: predicate === 'text', - }); const expectedText = split?.rest.join(' ').trim() ?? ''; const nextPositionals = [predicate, selectorExpression]; if (predicate === 'text' && expectedText.length > 0) { @@ -645,10 +648,7 @@ function collectReplaySelectorCandidates(action: SessionAction): string[] { } } if (action.command === 'is') { - const predicate = action.positionals?.[0]; - const split = splitSelectorFromArgs(action.positionals.slice(1), { - preferTrailingValue: predicate === 'text', - }); + const { split } = splitIsSelectorArgs(action.positionals); if (split) { result.push(split.selectorExpression); } diff --git a/src/daemon/selectors.ts b/src/daemon/selectors.ts index abcdb90fe..1540fede4 100644 --- a/src/daemon/selectors.ts +++ b/src/daemon/selectors.ts @@ -94,7 +94,7 @@ export function resolveSelectorChain( const diagnostics: SelectorDiagnostics[] = []; for (let i = 0; i < chain.selectors.length; i += 1) { const selector = chain.selectors[i]; - const summary = countSelectorMatches(nodes, selector, { + const summary = analyzeSelectorMatches(nodes, selector, { platform: options.platform, requireRect, }); @@ -102,10 +102,7 @@ export function resolveSelectorChain( if (summary.count === 0 || !summary.firstNode) continue; if (requireUnique && summary.count !== 1) { if (!disambiguateAmbiguous) continue; - const disambiguatedNode = pickDisambiguatedNode(nodes, selector, { - platform: options.platform, - requireRect, - }); + const disambiguatedNode = summary.disambiguated; if (!disambiguatedNode) continue; return { node: disambiguatedNode, @@ -138,13 +135,13 @@ export function findSelectorChainMatch( const diagnostics: SelectorDiagnostics[] = []; for (let i = 0; i < chain.selectors.length; i += 1) { const selector = chain.selectors[i]; - const summary = countSelectorMatches(nodes, selector, { + const matches = countSelectorMatchesOnly(nodes, selector, { platform: options.platform, requireRect, }); - diagnostics.push({ selector: selector.raw, matches: summary.count }); - if (summary.count > 0) { - return { selectorIndex: i, selector, matches: summary.count, diagnostics }; + diagnostics.push({ selector: selector.raw, matches }); + if (matches > 0) { + return { selectorIndex: i, selector, matches, diagnostics }; } } return null; @@ -197,9 +194,11 @@ export function splitSelectorFromArgs( if (boundaries.length === 0) return null; let boundary = boundaries[boundaries.length - 1]; if (preferTrailingValue) { - const boundaryWithRest = [...boundaries].reverse().find((index) => index < args.length); - if (boundaryWithRest !== undefined) { - boundary = boundaryWithRest; + for (let j = boundaries.length - 1; j >= 0; j -= 1) { + if (boundaries[j] < args.length) { + boundary = boundaries[j]; + break; + } } } const selectorExpression = args.slice(0, boundary).join(' ').trim(); @@ -210,6 +209,17 @@ export function splitSelectorFromArgs( }; } +export function splitIsSelectorArgs(positionals: string[]): { + predicate: string; + split: { selectorExpression: string; rest: string[] } | null; +} { + const predicate = positionals[0] ?? ''; + const split = splitSelectorFromArgs(positionals.slice(1), { + preferTrailingValue: predicate === 'text', + }); + return { predicate, split }; +} + export function isNodeVisible(node: SnapshotNode): boolean { if (node.hittable === true) return true; if (!node.rect) return false; @@ -455,13 +465,15 @@ function normalizeSelectorText(value: string | undefined): string | null { return trimmed; } -function countSelectorMatches( +function analyzeSelectorMatches( nodes: SnapshotState['nodes'], selector: Selector, options: { platform: 'ios' | 'android'; requireRect: boolean }, -): { count: number; firstNode: SnapshotNode | null } { +): { count: number; firstNode: SnapshotNode | null; disambiguated: SnapshotNode | null } { let count = 0; let firstNode: SnapshotNode | null = null; + let best: SnapshotNode | null = null; + let tie = false; for (const node of nodes) { if (options.requireRect && !node.rect) continue; if (!matchesSelector(node, selector, options.platform)) continue; @@ -469,20 +481,6 @@ function countSelectorMatches( if (!firstNode) { firstNode = node; } - } - return { count, firstNode }; -} - -function pickDisambiguatedNode( - nodes: SnapshotState['nodes'], - selector: Selector, - options: { platform: 'ios' | 'android'; requireRect: boolean }, -): SnapshotNode | null { - let best: SnapshotNode | null = null; - let tie = false; - for (const node of nodes) { - if (options.requireRect && !node.rect) continue; - if (!matchesSelector(node, selector, options.platform)) continue; if (!best) { best = node; tie = false; @@ -498,7 +496,25 @@ function pickDisambiguatedNode( tie = true; } } - return tie ? null : best; + return { + count, + firstNode, + disambiguated: tie ? null : best, + }; +} + +function countSelectorMatchesOnly( + nodes: SnapshotState['nodes'], + selector: Selector, + options: { platform: 'ios' | 'android'; requireRect: boolean }, +): number { + let count = 0; + for (const node of nodes) { + if (options.requireRect && !node.rect) continue; + if (!matchesSelector(node, selector, options.platform)) continue; + count += 1; + } + return count; } function compareDisambiguationCandidates(a: SnapshotNode, b: SnapshotNode): number { diff --git a/src/daemon/snapshot-processing.ts b/src/daemon/snapshot-processing.ts index a755d21dc..974967322 100644 --- a/src/daemon/snapshot-processing.ts +++ b/src/daemon/snapshot-processing.ts @@ -82,13 +82,9 @@ export function normalizeType(type: string): string { if (value.startsWith('ax')) { value = value.replace(/^ax/, ''); } - if (value.includes('.')) { - const suffix = value.split('.').pop(); - if (suffix) value = suffix; - } - if (value.includes('/')) { - const suffix = value.split('/').pop(); - if (suffix) value = suffix; + const lastSeparator = Math.max(value.lastIndexOf('.'), value.lastIndexOf('/')); + if (lastSeparator !== -1) { + value = value.slice(lastSeparator + 1); } return value; } From 7992add8baf8807cefb28eb77b68d180273b4e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Feb 2026 13:31:30 +0100 Subject: [PATCH 3/4] Heal numeric get-text drift in replay update --- .../handlers/__tests__/replay-heal.test.ts | 81 +++++++++++++++++++ src/daemon/handlers/interaction.ts | 1 + src/daemon/handlers/session.ts | 62 +++++++++++++- 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/src/daemon/handlers/__tests__/replay-heal.test.ts b/src/daemon/handlers/__tests__/replay-heal.test.ts index 4067d38dd..2b8512f33 100644 --- a/src/daemon/handlers/__tests__/replay-heal.test.ts +++ b/src/daemon/handlers/__tests__/replay-heal.test.ts @@ -365,6 +365,87 @@ test('replay --update heals selector in is command', async () => { assert.ok(rewrittenSelector.includes('auth_continue')); }); +test('replay --update heals numeric get text drift when numeric candidate value is unique', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-get-numeric-')); + const sessionsDir = path.join(tempRoot, 'sessions'); + const replayPath = path.join(tempRoot, 'replay.ad'); + const sessionStore = new SessionStore(sessionsDir); + const sessionName = 'heal-get-numeric-session'; + sessionStore.set(sessionName, makeSession(sessionName)); + + writeReplayFile(replayPath, { + ts: Date.now(), + command: 'get', + positionals: ['text', 'role="statictext" label="2" || label="2"'], + flags: {}, + result: {}, + }); + + const invokeCalls: string[] = []; + const invoke = async (request: DaemonRequest): Promise => { + if (request.command !== 'get') { + return { ok: false, error: { code: 'INVALID_ARGS', message: `unexpected command ${request.command}` } }; + } + const selector = request.positionals?.[1] ?? ''; + invokeCalls.push(selector); + if (selector.includes('label="2"')) { + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'selector stale' } }; + } + if (selector.includes('label="20"')) { + return { ok: true, data: { text: '20' } }; + } + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'unexpected selector' } }; + }; + + const dispatch = async (): Promise | void> => { + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeStaticText', + label: '20', + rect: { x: 0, y: 100, width: 100, height: 24 }, + enabled: true, + hittable: true, + }, + { + index: 1, + type: 'XCUIElementTypeStaticText', + label: 'Version: 0.84.0', + rect: { x: 0, y: 200, width: 220, height: 17 }, + enabled: true, + hittable: true, + }, + ], + truncated: false, + backend: 'xctest', + }; + }; + + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'replay', + positionals: [replayPath], + flags: { replayUpdate: true }, + }, + sessionName, + logPath: path.join(tempRoot, 'daemon.log'), + sessionStore, + invoke, + dispatch, + }); + + assert.ok(response); + assert.equal(response.ok, true, JSON.stringify(response)); + if (response.ok) { + assert.equal(response.data?.healed, 1); + assert.equal(response.data?.replayed, 1); + } + assert.equal(invokeCalls.length, 2); +}); + test('replay rejects legacy JSON payload files', async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-json-rejected-')); const sessionsDir = path.join(tempRoot, 'sessions'); diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index 74a1d8d5c..95507e77b 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -310,6 +310,7 @@ export async function handleInteractionCommands(params: { platform: session.device.platform, requireRect: false, requireUnique: true, + disambiguateAmbiguous: sub === 'text', }); if (!resolved) { return { diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 89de169ca..a2b4ba6e3 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -10,7 +10,7 @@ import { ensureDeviceReady } from '../device-ready.ts'; import { resolveIosAppStateFromSnapshots } from '../app-state.ts'; import { stopIosRunnerSession } from '../../platforms/ios/runner-client.ts'; import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; -import { pruneGroupNodes } from '../snapshot-processing.ts'; +import { extractNodeText, normalizeType, pruneGroupNodes } from '../snapshot-processing.ts'; import { buildSelectorChainForNode, resolveSelectorChain, @@ -517,6 +517,10 @@ async function healReplayAction(params: { const session = sessionStore.get(sessionName); if (!session) return null; const requiresRect = action.command === 'click' || action.command === 'fill'; + const allowDisambiguation = + action.command === 'click' || + action.command === 'fill' || + (action.command === 'get' && action.positionals?.[0] === 'text'); const snapshot = await captureSnapshotForReplay(session, action, logPath, requiresRect, dispatch, sessionStore); const selectorCandidates = collectReplaySelectorCandidates(action); for (const candidate of selectorCandidates) { @@ -526,7 +530,7 @@ async function healReplayAction(params: { platform: session.device.platform, requireRect: requiresRect, requireUnique: true, - disambiguateAmbiguous: requiresRect, + disambiguateAmbiguous: allowDisambiguation, }); if (!resolved) continue; const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, { @@ -580,6 +584,10 @@ async function healReplayAction(params: { }; } } + const numericDriftHeal = healNumericGetTextDrift(action, snapshot, session); + if (numericDriftHeal) { + return numericDriftHeal; + } return null; } @@ -697,6 +705,56 @@ function parseSelectorWaitPositionals(positionals: string[]): { }; } +function healNumericGetTextDrift( + action: SessionAction, + snapshot: SnapshotState, + session: SessionState, +): SessionAction | null { + if (action.command !== 'get') return null; + if (action.positionals?.[0] !== 'text') return null; + const selectorExpression = action.positionals?.[1]; + if (!selectorExpression) return null; + const chain = tryParseSelectorChain(selectorExpression); + if (!chain) return null; + + const roleFilters = new Set(); + let hasNumericTerm = false; + for (const selector of chain.selectors) { + for (const term of selector.terms) { + if (term.key === 'role' && typeof term.value === 'string') { + roleFilters.add(normalizeType(term.value)); + } + if ( + (term.key === 'text' || term.key === 'label' || term.key === 'value') && + typeof term.value === 'string' && + /^\d+$/.test(term.value.trim()) + ) { + hasNumericTerm = true; + } + } + } + if (!hasNumericTerm) return null; + + const numericNodes = snapshot.nodes.filter((node) => { + const text = extractNodeText(node).trim(); + if (!/^\d+$/.test(text)) return false; + if (roleFilters.size === 0) return true; + return roleFilters.has(normalizeType(node.type ?? '')); + }); + if (numericNodes.length === 0) return null; + const numericValues = uniqueStrings(numericNodes.map((node) => extractNodeText(node).trim())); + if (numericValues.length !== 1) return null; + + const targetNode = numericNodes.find((node) => extractNodeText(node).trim() === numericValues[0]); + if (!targetNode) return null; + const selectorChain = buildSelectorChainForNode(targetNode, session.device.platform, { action: 'get' }); + if (selectorChain.length === 0) return null; + return { + ...action, + positionals: ['text', selectorChain.join(' || ')], + }; +} + function parseReplayScript(script: string): SessionAction[] { const actions: SessionAction[] = []; const lines = script.split(/\r?\n/); From 2161788b21289a9b01bc0daaef8affb28b596281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Feb 2026 14:00:00 +0100 Subject: [PATCH 4/4] Remove redundant find and dead assignment in selector internals Co-authored-by: Cursor --- src/daemon/handlers/session.ts | 2 +- src/daemon/selectors.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index a2b4ba6e3..baf000a47 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -745,7 +745,7 @@ function healNumericGetTextDrift( const numericValues = uniqueStrings(numericNodes.map((node) => extractNodeText(node).trim())); if (numericValues.length !== 1) return null; - const targetNode = numericNodes.find((node) => extractNodeText(node).trim() === numericValues[0]); + const targetNode = numericNodes[0]; if (!targetNode) return null; const selectorChain = buildSelectorChainForNode(targetNode, session.device.platform, { action: 'get' }); if (selectorChain.length === 0) return null; diff --git a/src/daemon/selectors.ts b/src/daemon/selectors.ts index 1540fede4..2db1fc626 100644 --- a/src/daemon/selectors.ts +++ b/src/daemon/selectors.ts @@ -483,7 +483,6 @@ function analyzeSelectorMatches( } if (!best) { best = node; - tie = false; continue; } const comparison = compareDisambiguationCandidates(node, best);