From 51468ab1255e43da92bed99271862b380edb4fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Feb 2026 14:15:18 +0100 Subject: [PATCH 1/2] Improve Android open fallback, find ambiguity handling, and replay diagnostics --- .../handlers/__tests__/replay-heal.test.ts | 5 ++ src/daemon/handlers/find.ts | 22 +++++- src/daemon/handlers/session.ts | 44 ++++++++++- src/platforms/android/__tests__/index.test.ts | 17 +++++ src/platforms/android/index.ts | 76 +++++++++++++++---- src/utils/__tests__/finders.test.ts | 34 +++++++++ src/utils/finders.ts | 36 ++++++--- 7 files changed, 207 insertions(+), 27 deletions(-) create mode 100644 src/utils/__tests__/finders.test.ts diff --git a/src/daemon/handlers/__tests__/replay-heal.test.ts b/src/daemon/handlers/__tests__/replay-heal.test.ts index 61f1f0fee..5cea1fa65 100644 --- a/src/daemon/handlers/__tests__/replay-heal.test.ts +++ b/src/daemon/handlers/__tests__/replay-heal.test.ts @@ -225,6 +225,11 @@ test('replay without --update does not heal or rewrite', async () => { assert.ok(response); assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.equal(response.error.details?.step, 1); + assert.equal(response.error.details?.action, 'click'); + } assert.equal(snapshotDispatchCalls, 0); assert.equal(fs.readFileSync(replayPath, 'utf8'), originalPayload); }); diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 230d337c5..c55698bfa 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -1,5 +1,5 @@ import { dispatchCommand, resolveTargetDevice } from '../../core/dispatch.ts'; -import { findNodeByLocator, type FindLocator } from '../../utils/finders.ts'; +import { findBestMatchesByLocator, findNodeByLocator, type FindLocator } from '../../utils/finders.ts'; import { attachRefs, centerOfRect, type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; import { AppError } from '../../utils/errors.ts'; import type { DaemonRequest, DaemonResponse } from '../types.ts'; @@ -114,6 +114,26 @@ export async function handleFindCommands(params: { return { ok: false, error: { code: 'COMMAND_FAILED', message: 'find wait timed out' } }; } const { nodes } = await fetchNodes(); + const bestMatches = findBestMatchesByLocator(nodes, locator, query, { requireRect: requiresRect }); + if (requiresRect && bestMatches.matches.length > 1) { + const candidates = bestMatches.matches.slice(0, 8).map((candidate) => { + const label = extractNodeText(candidate) || candidate.label || candidate.identifier || candidate.type || ''; + return `@${candidate.ref}${label ? `(${label})` : ''}`; + }); + return { + ok: false, + error: { + code: 'AMBIGUOUS_MATCH', + message: `find matched ${bestMatches.matches.length} elements for ${locator} "${query}". Use a more specific locator or selector.`, + details: { + locator, + query, + matches: bestMatches.matches.length, + candidates, + }, + }, + }; + } const node = findNodeByLocator(nodes, locator, query, { requireRect: requiresRect }); if (!node) { return { ok: false, error: { code: 'COMMAND_FAILED', message: 'find did not match any element' } }; diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index ca89efbf7..211991e28 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -269,7 +269,9 @@ export async function handleSessionCommands(params: { flags: action.flags ?? {}, }); if (response.ok) continue; - if (!shouldUpdate) return response; + if (!shouldUpdate) { + return withReplayFailureContext(response, action, index, resolved); + } const nextAction = await healReplayAction({ action, sessionName, @@ -278,7 +280,7 @@ export async function handleSessionCommands(params: { dispatch, }); if (!nextAction) { - return response; + return withReplayFailureContext(response, action, index, resolved); } actions[index] = nextAction; response = await invoke({ @@ -289,7 +291,7 @@ export async function handleSessionCommands(params: { flags: nextAction.flags ?? {}, }); if (!response.ok) { - return response; + return withReplayFailureContext(response, nextAction, index, resolved); } healed += 1; } @@ -334,6 +336,42 @@ export async function handleSessionCommands(params: { return null; } +function withReplayFailureContext( + response: DaemonResponse, + action: SessionAction, + index: number, + replayPath: string, +): DaemonResponse { + if (response.ok) return response; + const step = index + 1; + const summary = formatReplayActionSummary(action); + const details = { + ...(response.error.details ?? {}), + replayPath, + step, + action: action.command, + positionals: action.positionals ?? [], + }; + return { + ok: false, + error: { + code: response.error.code, + message: `Replay failed at step ${step} (${summary}): ${response.error.message}`, + details, + }, + }; +} + +function formatReplayActionSummary(action: SessionAction): string { + const values = (action.positionals ?? []).map((value) => { + const trimmed = value.trim(); + if (/^-?\d+(\.\d+)?$/.test(trimmed)) return trimmed; + if (trimmed.startsWith('@')) return trimmed; + return JSON.stringify(trimmed); + }); + return [action.command, ...values].join(' '); +} + async function healReplayAction(params: { action: SessionAction; sessionName: string; diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index 30d382f03..f553837c7 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -1,5 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { parseAndroidLaunchComponent } from '../index.ts'; import { findBounds, parseUiHierarchy } from '../ui-hierarchy.ts'; test('parseUiHierarchy reads double-quoted Android node attributes', () => { @@ -72,3 +73,19 @@ test('findBounds ignores bounds-like fragments inside other attribute values', ( assert.deepEqual(findBounds(xml, 'target'), { x: 200, y: 350 }); }); + +test('parseAndroidLaunchComponent extracts final resolved component', () => { + const stdout = [ + 'priority=0 preferredOrder=0 match=0x108000 specificIndex=-1 isDefault=true', + 'com.boatsgroup.boattrader/com.boatsgroup.boattrader.MainActivity', + ].join('\n'); + assert.equal( + parseAndroidLaunchComponent(stdout), + 'com.boatsgroup.boattrader/com.boatsgroup.boattrader.MainActivity', + ); +}); + +test('parseAndroidLaunchComponent returns null when no component is present', () => { + const stdout = 'No activity found'; + assert.equal(parseAndroidLaunchComponent(stdout), null); +}); diff --git a/src/platforms/android/index.ts b/src/platforms/android/index.ts index aefa36b00..372f1a289 100644 --- a/src/platforms/android/index.ts +++ b/src/platforms/android/index.ts @@ -187,22 +187,70 @@ export async function openAndroidApp( ); return; } - await runCmd( + try { + await runCmd( + 'adb', + adbArgs(device, [ + 'shell', + 'am', + 'start', + '-a', + 'android.intent.action.MAIN', + '-c', + 'android.intent.category.DEFAULT', + '-c', + 'android.intent.category.LAUNCHER', + '-p', + resolved.value, + ]), + ); + return; + } catch (initialError) { + const component = await resolveAndroidLaunchComponent(device, resolved.value); + if (!component) throw initialError; + await runCmd( + 'adb', + adbArgs(device, [ + 'shell', + 'am', + 'start', + '-a', + 'android.intent.action.MAIN', + '-c', + 'android.intent.category.DEFAULT', + '-c', + 'android.intent.category.LAUNCHER', + '-n', + component, + ]), + ); + } +} + +async function resolveAndroidLaunchComponent( + device: DeviceInfo, + packageName: string, +): Promise { + const result = await runCmd( 'adb', - adbArgs(device, [ - 'shell', - 'am', - 'start', - '-a', - 'android.intent.action.MAIN', - '-c', - 'android.intent.category.DEFAULT', - '-c', - 'android.intent.category.LAUNCHER', - '-p', - resolved.value, - ]), + adbArgs(device, ['shell', 'cmd', 'package', 'resolve-activity', '--brief', packageName]), + { allowFailure: true }, ); + if (result.exitCode !== 0) return null; + return parseAndroidLaunchComponent(result.stdout); +} + +export function parseAndroidLaunchComponent(stdout: string): string | null { + const lines = stdout + .split('\n') + .map((line: string) => line.trim()) + .filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const line = lines[index]; + if (!line.includes('/')) continue; + return line.split(/\s+/)[0]; + } + return null; } export async function openAndroidDevice(device: DeviceInfo): Promise { diff --git a/src/utils/__tests__/finders.test.ts b/src/utils/__tests__/finders.test.ts new file mode 100644 index 000000000..d39a90fc7 --- /dev/null +++ b/src/utils/__tests__/finders.test.ts @@ -0,0 +1,34 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { findBestMatchesByLocator, findNodeByLocator } from '../finders.ts'; +import type { SnapshotNode } from '../snapshot.ts'; + +function makeNode(ref: string, label?: string, identifier?: string): SnapshotNode { + return { + index: Number(ref.replace('e', '')) || 0, + ref, + type: 'android.widget.TextView', + label, + identifier, + rect: { x: 0, y: 0, width: 100, height: 20 }, + }; +} + +test('findBestMatchesByLocator returns all best-scored matches', () => { + const nodes: SnapshotNode[] = [ + makeNode('e1', 'Continue'), + makeNode('e2', 'Continue'), + makeNode('e3', 'Continue later'), + ]; + const result = findBestMatchesByLocator(nodes, 'label', 'Continue', { requireRect: true }); + assert.equal(result.score, 2); + assert.equal(result.matches.length, 2); + assert.equal(result.matches[0]?.ref, 'e1'); + assert.equal(result.matches[1]?.ref, 'e2'); +}); + +test('findNodeByLocator preserves first best match behavior', () => { + const nodes: SnapshotNode[] = [makeNode('e1', 'Continue'), makeNode('e2', 'Continue')]; + const match = findNodeByLocator(nodes, 'label', 'Continue', { requireRect: true }); + assert.equal(match?.ref, 'e1'); +}); diff --git a/src/utils/finders.ts b/src/utils/finders.ts index d32e843d4..bc4cd2561 100644 --- a/src/utils/finders.ts +++ b/src/utils/finders.ts @@ -6,28 +6,46 @@ export type FindMatchOptions = { requireRect?: boolean; }; +export type FindBestMatches = { + matches: SnapshotNode[]; + score: number; +}; + export function findNodeByLocator( nodes: SnapshotNode[], locator: FindLocator, query: string, options: FindMatchOptions = {}, ): SnapshotNode | null { + const best = findBestMatchesByLocator(nodes, locator, query, options); + return best.matches[0] ?? null; +} + +export function findBestMatchesByLocator( + nodes: SnapshotNode[], + locator: FindLocator, + query: string, + options: FindMatchOptions = {}, +): FindBestMatches { const normalizedQuery = normalizeText(query); - if (!normalizedQuery) return null; - let best: { node: SnapshotNode; score: number } | null = null; + if (!normalizedQuery) return { matches: [], score: 0 }; + let bestScore = 0; + const matches: SnapshotNode[] = []; for (const node of nodes) { if (options.requireRect && !node.rect) continue; const score = matchNode(node, locator, normalizedQuery); if (score <= 0) continue; - if (!best || score > best.score) { - best = { node, score }; - if (score >= 2) { - // exact match, keep first exact match - break; - } + if (score > bestScore) { + bestScore = score; + matches.length = 0; + matches.push(node); + continue; + } + if (score === bestScore) { + matches.push(node); } } - return best?.node ?? null; + return { matches, score: bestScore }; } function matchNode(node: SnapshotNode, locator: FindLocator, query: string): number { From 0b063302499ef6e54b2028449ee55dce06a8d4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Feb 2026 14:39:13 +0100 Subject: [PATCH 2/2] Fix iOS integration step to use unique selector for General --- test/integration/ios.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/ios.test.ts b/test/integration/ios.test.ts index 3df44e0f3..7bc917ee3 100644 --- a/test/integration/ios.test.ts +++ b/test/integration/ios.test.ts @@ -30,14 +30,14 @@ test('ios settings commands', { skip: shouldSkipIos() }, async () => { detail: 'expected snapshot to include a nodes array', }); - const openGeneralArgs = ['find', 'text', 'General', 'click', '--json', ...session]; + const openGeneralArgs = ['click', 'role=cell', 'label=General', '--json', ...session]; const openGeneral = integration.runStep('open general', openGeneralArgs); integration.assertResult( openGeneral.json?.success, 'open general success', openGeneralArgs, openGeneral, - { detail: 'expected find General click to return success=true' }, + { detail: 'expected click role=cell label=General to return success=true' }, ); const snapshotGeneralArgs = ['snapshot', '--json', ...session];