diff --git a/src/command-catalog.ts b/src/command-catalog.ts index 9279926ac..9dc44851f 100644 --- a/src/command-catalog.ts +++ b/src/command-catalog.ts @@ -110,6 +110,26 @@ export const DAEMON_COMMAND_GROUPS = { PUBLIC_COMMANDS.type, PUBLIC_COMMANDS.wait, ), + androidBlockingDialogGuardedAction: commandSet( + PUBLIC_COMMANDS.back, + PUBLIC_COMMANDS.click, + PUBLIC_COMMANDS.fill, + PUBLIC_COMMANDS.focus, + PUBLIC_COMMANDS.gesture, + PUBLIC_COMMANDS.home, + PUBLIC_COMMANDS.keyboard, + PUBLIC_COMMANDS.longPress, + 'fling', + 'pan', + 'pinch', + PUBLIC_COMMANDS.press, + PUBLIC_COMMANDS.rotate, + 'rotate-gesture', + PUBLIC_COMMANDS.scroll, + PUBLIC_COMMANDS.swipe, + 'transform-gesture', + PUBLIC_COMMANDS.type, + ), selectorValidationExempt: commandSet( INTERNAL_COMMANDS.sessionList, PUBLIC_COMMANDS.devices, diff --git a/src/daemon/__tests__/request-router-android-modal.test.ts b/src/daemon/__tests__/request-router-android-modal.test.ts index a01a2ddb2..2af93c22a 100644 --- a/src/daemon/__tests__/request-router-android-modal.test.ts +++ b/src/daemon/__tests__/request-router-android-modal.test.ts @@ -56,6 +56,7 @@ vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => { ...actual, openAndroidApp: vi.fn(async () => {}), getAndroidAppState: vi.fn(async () => ({ package: 'com.android.settings' })), + getAndroidBlockingDialogFocus: vi.fn(async () => null), }; }); diff --git a/src/daemon/__tests__/request-router-screenshot.test.ts b/src/daemon/__tests__/request-router-screenshot.test.ts index d86be9a8b..8d67bb4d3 100644 --- a/src/daemon/__tests__/request-router-screenshot.test.ts +++ b/src/daemon/__tests__/request-router-screenshot.test.ts @@ -8,6 +8,14 @@ vi.mock('../../core/dispatch.ts', async (importOriginal) => { return { ...actual, dispatchCommand: vi.fn(async () => ({})) }; }); +vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getAndroidBlockingDialogFocus: vi.fn(async () => null), + }; +}); + import { dispatchCommand } from '../../core/dispatch.ts'; import { createRequestHandler } from '../request-router.ts'; import { dispatchScreenshotViaRuntime } from '../screenshot-runtime.ts'; diff --git a/src/daemon/android-system-dialog.ts b/src/daemon/android-system-dialog.ts index 1af5a5cfe..a06507d56 100644 --- a/src/daemon/android-system-dialog.ts +++ b/src/daemon/android-system-dialog.ts @@ -1,18 +1,37 @@ -import { getAndroidAppState, openAndroidApp } from '../platforms/android/app-lifecycle.ts'; +import { + getAndroidAppState, + getAndroidBlockingDialogFocus, + openAndroidApp, + type AndroidBlockingDialogFocus, +} from '../platforms/android/app-lifecycle.ts'; import { snapshotAndroid } from '../platforms/android/snapshot.ts'; import { runAndroidAdb } from '../platforms/android/adb.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; +import { AppError } from '../utils/errors.ts'; import { centerOfRect, attachRefs, type SnapshotNode } from '../utils/snapshot.ts'; import { sleep } from '../utils/timeouts.ts'; import { pruneGroupNodes } from './snapshot-processing.ts'; import type { SessionState } from './types.ts'; -const ANDROID_BLOCKING_MODAL_PATTERN = /\bis(?:n't| not)\s+responding\b/i; +const ANDROID_BLOCKING_MODAL_PATTERN = /\bis(?:n(?:'|'|')?t| not)\s+responding\b/i; const ANDROID_CLOSE_APP_PATTERN = /^close app$/i; const ANDROID_MODAL_POLL_MS = 500; const ANDROID_MODAL_POLL_ATTEMPTS = 12; +const ANDROID_BLOCKING_DIALOG_HINT = + 'Wait for Android to recover, close the dialog, restart the app, or reboot the emulator, then retry.'; export type AndroidBlockingDialogRecoveryResult = 'absent' | 'recovered' | 'failed'; +export type AndroidBlockingDialogReadinessResult = + | { status: 'clear' } + | { status: 'recovered'; warning: string }; +type AndroidDialogButtonTapResult = + | { ok: true; x: number; y: number } + | { + ok: false; + exitCode: number; + stdout: string; + stderr: string; + }; export async function recoverAndroidBlockingSystemDialog(params: { session: SessionState; @@ -30,13 +49,8 @@ export async function recoverAndroidBlockingSystemDialog(params: { return 'absent'; } - const { x, y } = centerOfRect(closeAppButton.rect); - const tapResult = await runAndroidAdb( - session.device, - ['shell', 'input', 'tap', String(Math.round(x)), String(Math.round(y))], - { allowFailure: true }, - ); - if (tapResult.exitCode !== 0) { + const tapResult = await tapAndroidDialogButton(session, closeAppButton); + if (!tapResult.ok) { emitDiagnostic({ level: 'warn', phase: 'android_blocking_dialog_tap_failed', @@ -66,7 +80,7 @@ export async function recoverAndroidBlockingSystemDialog(params: { if (session.appBundleId) { await openAndroidApp(session.device, session.appBundleId); - const focused = await waitForFocusedAndroidApp(session, session.appBundleId); + const focused = await waitForAndroidAppFocus(session, session.appBundleId); if (!focused) { emitDiagnostic({ level: 'warn', @@ -88,8 +102,8 @@ export async function recoverAndroidBlockingSystemDialog(params: { session: session.name, deviceId: session.device.id, appBundleId: session.appBundleId, - x, - y, + x: tapResult.x, + y: tapResult.y, }, }); return 'recovered'; @@ -107,6 +121,126 @@ export async function recoverAndroidBlockingSystemDialog(params: { } } +export async function ensureAndroidBlockingSystemDialogReady(params: { + session: SessionState; + command: string; + phase: 'before-command' | 'after-command'; +}): Promise { + const { session, command } = params; + if (session.device.platform !== 'android') return { status: 'clear' }; + + const focus = await getAndroidBlockingDialogFocus(session.device); + if (!focus) return { status: 'clear' }; + + if (isSessionAppAnr(session, focus)) { + const recovered = await recoverAppOwnedAndroidBlockingSystemDialogSafely(session); + if (recovered) { + const warning = `Recovered Android app ANR before ${command}: closed and relaunched ${session.appBundleId}.`; + if (params.phase === 'before-command') return { status: 'recovered', warning }; + + throw androidBlockingDialogError({ + session, + command, + focus, + message: `Android app ANR appeared after ${command}; ${session.appBundleId} was closed and relaunched. Retry the command against the fresh app session.`, + hint: 'Retry the command. If the ANR returns, inspect app logs or restart the emulator.', + }); + } + + throw androidBlockingDialogError({ + session, + command, + focus, + message: `Android app ANR blocked ${command}: ${formatAndroidBlockingDialogFocus(focus)}. Automatic recovery failed.`, + hint: ANDROID_BLOCKING_DIALOG_HINT, + }); + } + + throw androidBlockingDialogError({ + session, + command, + focus, + message: `Android system dialog is blocking ${command}: ${formatAndroidBlockingDialogFocus(focus)}.`, + hint: ANDROID_BLOCKING_DIALOG_HINT, + }); +} + +async function recoverAppOwnedAndroidBlockingSystemDialogSafely( + session: SessionState, +): Promise { + try { + return await recoverAppOwnedAndroidBlockingSystemDialog(session); + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'android_app_anr_recovery_failed', + data: { + session: session.name, + deviceId: session.device.id, + appBundleId: session.appBundleId, + error: error instanceof Error ? error.message : String(error), + }, + }); + return false; + } +} + +function isSessionAppAnr(session: SessionState, focus: AndroidBlockingDialogFocus): boolean { + return Boolean(session.appBundleId && focus.package === session.appBundleId); +} + +async function recoverAppOwnedAndroidBlockingSystemDialog(session: SessionState): Promise { + if (!session.appBundleId) return false; + + const nodes = await readAndroidSnapshotNodes(session); + const closeAppButton = findCloseAppButton(nodes, { requireDialogSignal: false }); + if (!closeAppButton?.rect) return false; + + const tapResult = await tapAndroidDialogButton(session, closeAppButton); + if (!tapResult.ok) return false; + + await openAndroidApp(session.device, session.appBundleId); + const focused = await waitForAndroidAppFocus(session, session.appBundleId, { + requireNoBlockingDialog: true, + }); + if (focused) { + emitDiagnostic({ + level: 'warn', + phase: 'android_app_anr_recovered', + data: { + session: session.name, + deviceId: session.device.id, + appBundleId: session.appBundleId, + x: tapResult.x, + y: tapResult.y, + }, + }); + } + return focused; +} + +function androidBlockingDialogError(params: { + session: SessionState; + command: string; + focus: AndroidBlockingDialogFocus; + message: string; + hint: string; +}): AppError { + const { session, command, focus, message, hint } = params; + return new AppError('COMMAND_FAILED', message, { + command, + expectedPackage: session.appBundleId, + focusedPackage: focus.package, + focusedWindow: focus.focusedWindow, + rawFocus: focus.raw, + hint, + }); +} + +function formatAndroidBlockingDialogFocus(focus: AndroidBlockingDialogFocus): string { + return focus.package ? `${focus.focusedWindow} (package ${focus.package})` : focus.focusedWindow; +} + async function readAndroidSnapshotNodes(session: SessionState): Promise { const rawSnapshot = await snapshotAndroid(session.device, { interactiveOnly: false, @@ -115,13 +249,41 @@ async function readAndroidSnapshotNodes(session: SessionState): Promise { + if (!button.rect) { + return { ok: false, exitCode: 1, stdout: '', stderr: 'button has no rect' }; + } + const { x, y } = centerOfRect(button.rect); + const result = await runAndroidAdb( + session.device, + ['shell', 'input', 'tap', String(Math.round(x)), String(Math.round(y))], + { allowFailure: true }, + ); + if (result.exitCode !== 0) { + return { + ok: false, + exitCode: result.exitCode, + stdout: result.stdout.trim(), + stderr: result.stderr.trim(), + }; + } + return { ok: true, x, y }; +} + +function findCloseAppButton( + nodes: SnapshotNode[], + options: { requireDialogSignal?: boolean } = {}, +): SnapshotNode | undefined { + if (options.requireDialogSignal !== false && !containsBlockingDialog(nodes)) { return undefined; } return nodes.find((node) => { - const text = readNodeText(node); - return text.length > 0 && ANDROID_CLOSE_APP_PATTERN.test(text) && node.rect; + return ( + readNodeTextParts(node).some((text) => ANDROID_CLOSE_APP_PATTERN.test(text)) && node.rect + ); }); } @@ -137,17 +299,28 @@ async function waitForBlockingDialogToDismiss(session: SessionState): Promise { for (let attempt = 0; attempt < ANDROID_MODAL_POLL_ATTEMPTS; attempt += 1) { - const state = await getAndroidAppState(session.device); - if (state.package === appBundleId) { + if (await isAndroidAppFocused(session, appBundleId, options)) { return true; } await sleep(ANDROID_MODAL_POLL_MS); } + return await isAndroidAppFocused(session, appBundleId, options); +} + +async function isAndroidAppFocused( + session: SessionState, + appBundleId: string, + options: { requireNoBlockingDialog?: boolean }, +): Promise { + if (options.requireNoBlockingDialog && (await getAndroidBlockingDialogFocus(session.device))) { + return false; + } const state = await getAndroidAppState(session.device); return state.package === appBundleId; } @@ -157,14 +330,21 @@ function readNodeText(node: { value?: string | number | boolean | null; identifier?: string; }): string { + return readNodeTextParts(node).join(' ').trim(); +} + +function readNodeTextParts(node: { + label?: string; + value?: string | number | boolean | null; + identifier?: string; +}): string[] { const parts = [node.label, node.identifier]; if (typeof node.value === 'string' && node.value.trim().length > 0) { parts.push(node.value); } return parts .filter((part): part is string => typeof part === 'string' && part.trim().length > 0) - .join(' ') - .trim(); + .map((part) => part.trim()); } function containsBlockingDialog(nodes: SnapshotNode[]): boolean { diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 60e4eb7d9..b2340af35 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -40,6 +40,7 @@ vi.mock('../../../platforms/android/app-lifecycle.ts', async (importOriginal) => return { ...actual, getAndroidAppState: vi.fn(async () => ({})), + getAndroidBlockingDialogFocus: vi.fn(async () => null), }; }); @@ -64,11 +65,15 @@ vi.mock('../../../platforms/ios/runner-client.ts', async (importOriginal) => { }); import { dispatchCommand } from '../../../core/dispatch.ts'; -import { getAndroidAppState } from '../../../platforms/android/app-lifecycle.ts'; +import { + getAndroidAppState, + getAndroidBlockingDialogFocus, +} from '../../../platforms/android/app-lifecycle.ts'; import { getAndroidScreenSize } from '../../../platforms/android/input-actions.ts'; import { captureSnapshotForSession } from '../interaction-snapshot.ts'; const mockDispatch = vi.mocked(dispatchCommand); const mockGetAndroidAppState = vi.mocked(getAndroidAppState); +const mockGetAndroidBlockingDialogFocus = vi.mocked(getAndroidBlockingDialogFocus); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotForSession); @@ -132,6 +137,8 @@ beforeEach(() => { mockDispatch.mockResolvedValue({}); mockGetAndroidAppState.mockReset(); mockGetAndroidAppState.mockResolvedValue({}); + mockGetAndroidBlockingDialogFocus.mockReset(); + mockGetAndroidBlockingDialogFocus.mockResolvedValue(null); mockGetAndroidScreenSize.mockReset(); mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 }); mockCaptureSnapshotForSession.mockReset(); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 9c46354ff..494722171 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -46,6 +46,7 @@ import { readSimpleIosSelectorTarget, type DirectIosSelectorTarget, } from '../direct-ios-selector.ts'; +import { ensureAndroidBlockingSystemDialogReady } from '../android-system-dialog.ts'; export async function handleTouchInteractionCommands( params: InteractionHandlerParams & { @@ -429,10 +430,24 @@ async function dispatchRuntimeInteraction< const runtime = createInteractionRuntime(params); const actionStartedAt = Date.now(); try { + const readiness = await ensureAndroidBlockingSystemDialogReady({ + session, + command: params.req.command, + phase: 'before-command', + }); const runtimeResult = await options.run(runtime); await options.afterRun?.(runtimeResult); + await ensureAndroidBlockingSystemDialogReady({ + session, + command: params.req.command, + phase: 'after-command', + }); const actionFinishedAt = Date.now(); const { result, responseData } = await options.buildPayloads(runtimeResult); + if (readiness.status === 'recovered') { + result.warning = readiness.warning; + responseData.warning = readiness.warning; + } return finalizeTouchInteraction({ session, sessionStore: params.sessionStore, diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index 99c595ff7..dba9101d1 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -1,4 +1,4 @@ -import type { DaemonResponse } from '../types.ts'; +import type { DaemonResponse, SessionState } from '../types.ts'; import type { InteractionHandlerParams } from './interaction-common.ts'; import { handleTouchInteractionCommands } from './interaction-touch.ts'; import { captureSnapshotForSession } from './interaction-snapshot.ts'; @@ -11,7 +11,10 @@ import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; import { typeCommandDefinition } from '../../commands/interactions/definition.ts'; import { normalizeError } from '../../utils/errors.ts'; import { successText } from '../../utils/success-text.ts'; -import { recoverAndroidBlockingSystemDialog } from '../android-system-dialog.ts'; +import { + ensureAndroidBlockingSystemDialogReady, + recoverAndroidBlockingSystemDialog, +} from '../android-system-dialog.ts'; export async function handleInteractionCommands( params: InteractionHandlerParams, @@ -45,35 +48,64 @@ async function dispatchTypeViaRuntime( captureSnapshotForSession: typeof captureSnapshotForSession; }, ): Promise { - const { req, sessionName, sessionStore } = params; + const { sessionName, sessionStore } = params; const session = sessionStore.get(sessionName); if (!session) return errorResponse('SESSION_NOT_FOUND', 'No active session. Run open first.'); if (!isCommandSupportedOnDevice(typeCommandDefinition.name, session.device)) { return errorResponse('UNSUPPORTED_OPERATION', 'type is not supported on this device'); } + const recordingRecoveryResponse = await recoverAndroidRecordingDialogForType(session); + if (recordingRecoveryResponse) return recordingRecoveryResponse; + + return await runTypeTextViaRuntime(params, session); +} + +async function recoverAndroidRecordingDialogForType( + session: SessionState, +): Promise { if (session.device.platform === 'android' && session.recording) { const androidRecoveryResult = await recoverAndroidBlockingSystemDialog({ session }); if (androidRecoveryResult === 'failed') { return errorResponse('COMMAND_FAILED', 'Android system dialog blocked the recording session'); } } + return null; +} +async function runTypeTextViaRuntime( + params: InteractionHandlerParams & { + captureSnapshotForSession: typeof captureSnapshotForSession; + }, + session: SessionState, +): Promise { + const { req, sessionName, sessionStore } = params; const text = (req.positionals ?? []).join(' '); const runtime = createInteractionRuntime(params); const actionStartedAt = Date.now(); try { + const readiness = await ensureAndroidBlockingSystemDialogReady({ + session, + command: req.command, + phase: 'before-command', + }); const result = await runtime.interactions.typeText(text, { session: sessionName, requestId: req.meta?.requestId, delayMs: req.flags?.delayMs, }); + await ensureAndroidBlockingSystemDialogReady({ + session, + command: req.command, + phase: 'after-command', + }); const actionFinishedAt = Date.now(); - const responseData = { + const responseData: Record = { ...(result.backendResult ?? {}), text: result.text, delayMs: result.delayMs, ...successText(result.message ?? `Typed ${Array.from(result.text).length} chars`), }; + if (readiness.status === 'recovered') responseData.warning = readiness.warning; return finalizeTouchInteraction({ session, sessionStore, diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index db4a10c19..f8c84e4fc 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -1,5 +1,5 @@ import { dispatchCommand, type CommandFlags } from '../core/dispatch.ts'; -import { GESTURE_SUBCOMMAND_ERROR } from '../command-catalog.ts'; +import { DAEMON_COMMAND_GROUPS, GESTURE_SUBCOMMAND_ERROR } from '../command-catalog.ts'; import { isCommandSupportedOnDevice } from '../core/capabilities.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonCommandContext } from './context.ts'; @@ -10,7 +10,10 @@ import { dispatchScreenshotViaRuntime, type ScreenshotOutputPlacement, } from './screenshot-runtime.ts'; -import { recoverAndroidBlockingSystemDialog } from './android-system-dialog.ts'; +import { + ensureAndroidBlockingSystemDialogReady, + recoverAndroidBlockingSystemDialog, +} from './android-system-dialog.ts'; import { annotateScreenshotWithRefs } from './screenshot-overlay.ts'; import { isNavigationSensitiveAction, @@ -21,6 +24,7 @@ import { recordTouchVisualizationEvent, } from './recording-gestures.ts'; import { markPostGestureStabilization } from './post-gesture-stabilization.ts'; +import { normalizeError } from '../utils/errors.ts'; const GESTURE_PLATFORM_COMMANDS: Readonly> = { pan: 'pan', @@ -57,6 +61,8 @@ export async function dispatchGenericCommand(params: { const readinessResponse = await ensureGenericCommandReady(session, platformCommand); if (readinessResponse) return readinessResponse; + const preflightReadiness = await ensureNoAndroidBlockingDialogReady(session, platformCommand); + if ('response' in preflightReadiness) return preflightReadiness.response; const { resolvedPositionals, resolvedOut, recordedPositionals, recordedFlags } = resolveCommandPositionals(dispatchRequest); @@ -66,7 +72,7 @@ export async function dispatchGenericCommand(params: { ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), surface: session.surface, }; - const data = await executeGenericPlatformCommand({ + let data = await executeGenericPlatformCommand({ session, sessionName: params.sessionName, logPath, @@ -76,6 +82,20 @@ export async function dispatchGenericCommand(params: { out: resolvedOut, dispatchContext, }); + const postflightReadiness = await ensureNoAndroidBlockingDialogReady( + session, + platformCommand, + 'after-command', + ); + if ('response' in postflightReadiness) return postflightReadiness.response; + if ( + 'status' in preflightReadiness && + preflightReadiness.status === 'recovered' && + (!data || typeof data === 'object') + ) { + data ??= {}; + data.warning = preflightReadiness.warning; + } const actionFinishedAt = Date.now(); const actionRecordedPositionals = @@ -104,6 +124,30 @@ export async function dispatchGenericCommand(params: { return { ok: true, data: data ?? {} }; } +async function ensureNoAndroidBlockingDialogReady( + session: SessionState, + platformCommand: string, + phase: 'before-command' | 'after-command' = 'before-command', +): Promise< + { status: 'clear' } | { status: 'recovered'; warning: string } | { response: DaemonResponse } +> { + if ( + session.device.platform !== 'android' || + !DAEMON_COMMAND_GROUPS.androidBlockingDialogGuardedAction.has(platformCommand) + ) { + return { status: 'clear' }; + } + try { + return await ensureAndroidBlockingSystemDialogReady({ + session, + command: platformCommand, + phase, + }); + } catch (error) { + return { response: { ok: false, error: normalizeError(error) } }; + } +} + async function ensureGenericCommandReady( session: SessionState, platformCommand: string, diff --git a/src/platforms/android/app-lifecycle.ts b/src/platforms/android/app-lifecycle.ts index 2e3db407e..fb1ccb118 100644 --- a/src/platforms/android/app-lifecycle.ts +++ b/src/platforms/android/app-lifecycle.ts @@ -19,8 +19,10 @@ import { classifyAndroidAppTarget } from './open-target.ts'; import { prepareAndroidInstallArtifact } from './install-artifact.ts'; import { parseAndroidForegroundApp, + parseAndroidBlockingDialogFocus, parseAndroidLaunchablePackages, parseAndroidUserInstalledPackages, + type AndroidBlockingDialogFocus, type AndroidForegroundApp, } from './app-parsers.ts'; @@ -28,6 +30,7 @@ export { parseAndroidForegroundApp, parseAndroidLaunchablePackages, parseAndroidUserInstalledPackages, + type AndroidBlockingDialogFocus, type AndroidForegroundApp, } from './app-parsers.ts'; @@ -210,6 +213,15 @@ export async function getAndroidAppState(device: DeviceInfo): Promise { + return await readAndroidBlockingDialogFocus(device, [ + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'dumpsys', 'window'], + ]); +} + async function readAndroidFocus( device: DeviceInfo, commands: string[][], @@ -223,6 +235,18 @@ async function readAndroidFocus( return null; } +async function readAndroidBlockingDialogFocus( + device: DeviceInfo, + commands: string[][], +): Promise { + for (const args of commands) { + const result = await runAndroidAdb(device, args, { allowFailure: true }); + const parsed = parseAndroidBlockingDialogFocus(result.stdout ?? ''); + if (parsed) return parsed; + } + return null; +} + function androidLocalhostReverseEndpoint(target: string): AndroidPortReverseEndpoint | null { let url: URL; try { diff --git a/src/platforms/android/app-parsers.ts b/src/platforms/android/app-parsers.ts index a6b72b0bd..909fca238 100644 --- a/src/platforms/android/app-parsers.ts +++ b/src/platforms/android/app-parsers.ts @@ -1,4 +1,19 @@ export type AndroidForegroundApp = { package?: string; activity?: string }; +export type AndroidBlockingDialogFocus = { + package?: string; + focusedWindow: string; + raw: string; +}; + +const ANDROID_FOCUS_MARKERS = [ + 'mCurrentFocus=Window{', + 'mFocusedApp=AppWindowToken{', + 'mResumedActivity:', + 'ResumedActivity:', +] as const; +const ANDROID_ANR_TITLE_PATTERN = /\bApplication Not Responding:\s*([A-Za-z0-9_.]+)/i; +const ANDROID_RESPONDING_TITLE_PATTERN = /([^{}]*\bis(?:n't| not)\s+responding[^{}]*)/i; +const ANDROID_PACKAGE_PATTERN = /\b([A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)+)\b/; export function parseAndroidLaunchablePackages(stdout: string): string[] { const packages = new Set(); @@ -25,26 +40,61 @@ export function parseAndroidUserInstalledPackages(stdout: string): string[] { } export function parseAndroidForegroundApp(text: string): AndroidForegroundApp | null { - const markers = [ - 'mCurrentFocus=Window{', - 'mFocusedApp=AppWindowToken{', - 'mResumedActivity:', - 'ResumedActivity:', - ]; + return parseAndroidFocusSegment(text, (segment) => parseAndroidComponentFromSegment(segment)); +} + +export function parseAndroidBlockingDialogFocus(text: string): AndroidBlockingDialogFocus | null { + return parseAndroidFocusSegment(text, (segment, raw) => + parseAndroidBlockingDialogFromSegment(segment, raw), + ); +} + +function parseAndroidFocusSegment( + text: string, + parse: (segment: string, raw: string) => T | null, +): T | null { const lines = text.split('\n'); - for (const marker of markers) { + for (const marker of ANDROID_FOCUS_MARKERS) { for (const line of lines) { const markerIndex = line.indexOf(marker); if (markerIndex === -1) continue; + const raw = line.trim(); const segment = line.slice(markerIndex + marker.length); - const parsed = parseAndroidComponentFromSegment(segment); + const parsed = parse(segment, raw); if (parsed) return parsed; } } return null; } +function parseAndroidBlockingDialogFromSegment( + segment: string, + raw: string, +): AndroidBlockingDialogFocus | null { + const windowText = segment.split('}')[0]?.trim() ?? segment.trim(); + const anrMatch = ANDROID_ANR_TITLE_PATTERN.exec(windowText); + if (anrMatch) { + const packageName = anrMatch[1]; + return { + package: packageName, + focusedWindow: `Application Not Responding: ${packageName}`, + raw, + }; + } + + const respondingMatch = ANDROID_RESPONDING_TITLE_PATTERN.exec(windowText); + if (!respondingMatch) return null; + + const focusedWindow = respondingMatch[1].trim().replace(/\s+/g, ' '); + const packageName = ANDROID_PACKAGE_PATTERN.exec(focusedWindow)?.[1]; + return { + ...(packageName ? { package: packageName } : {}), + focusedWindow, + raw, + }; +} + function parseAndroidComponentFromSegment(segment: string): AndroidForegroundApp | null { for (const token of segment.trim().split(/\s+/)) { const slashIndex = token.indexOf('/'); diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 634d202cb..a97c8957b 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -3,7 +3,13 @@ import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; import type { AgentDeviceClient } from '../../../src/client-types.ts'; -import { arrayEqual, assertCommandCall, assertPngFile } from './assertions.ts'; +import { + arrayEqual, + assertCommandCall, + assertPngFile, + assertRpcError, + assertRpcOk, +} from './assertions.ts'; import { createAndroidSettingsWorld, waitForFileContent } from './android-world.ts'; import { PROVIDER_SCENARIO_ANDROID } from './fixtures.ts'; import { createProviderScenarioTempPath, withProviderScenarioResource } from './harness.ts'; @@ -230,6 +236,80 @@ test('Provider-backed integration Android alert handles system dialogs', async ( ); }); +test('Provider-backed integration Android app-owned ANR recovers before action commands', async () => { + let anrFocused = true; + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + snapshotXml: () => (anrFocused ? androidSystemDialogXml() : androidAppOwnedSheetXml()), + dumpsysWindow: () => + anrFocused + ? 'mCurrentFocus=Window{7f8 u0 Application Not Responding: com.example.demo}\n' + : 'mCurrentFocus=Window{42 u0 com.example.demo/.MainActivity}\n', + onAdbExec: (args) => { + if (args[0] === 'shell' && args[1] === 'input' && args[2] === 'tap' && anrFocused) { + anrFocused = false; + } + }, + }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const press = await world.daemon.callCommand('press', ['50', '60'], world.selection); + if (press.json?.error) { + assert.fail(JSON.stringify({ response: press.json, adbCalls: world.adbCalls }, null, 2)); + } + const pressData = assertRpcOk(press); + + assert.equal(pressData.x, 50); + assert.equal(pressData.y, 60); + assert.match(String(pressData.warning ?? ''), /Recovered Android app ANR before press/); + assertCommandCall(world.adbCalls, ['shell', 'input', 'tap', '116', '638']); + assertCommandCall(world.adbCalls, ['shell', 'input', 'tap', '50', '60']); + assert.ok( + world.adbCalls.filter((call) => call.slice(0, 4).join(' ') === 'shell am start -W') + .length >= 2, + JSON.stringify(world.adbCalls), + ); + }, + ); +}); + +test('Provider-backed integration Android external ANR fails with actionable context', async () => { + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + snapshotXml: androidSystemDialogXml, + dumpsysWindow: () => + 'mCurrentFocus=Window{7f8 u0 Application Not Responding: com.android.systemui}\n', + }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const openCalls = world.adbCalls.filter( + (call) => call.slice(0, 4).join(' ') === 'shell am start -W', + ).length; + const press = await world.daemon.callCommand('press', ['50', '60'], world.selection); + const error = assertRpcError(press, 'COMMAND_FAILED', /com\.android\.systemui/); + const details = error.details as Record; + + assert.equal(details.focusedPackage, 'com.android.systemui'); + assert.equal(details.expectedPackage, 'com.example.demo'); + assert.equal( + world.adbCalls.some((call) => call.join(' ') === 'shell input tap 116 638'), + false, + JSON.stringify(world.adbCalls), + ); + assert.equal( + world.adbCalls.filter((call) => call.slice(0, 4).join(' ') === 'shell am start -W').length, + openCalls, + ); + }, + ); +}); + test('Provider-backed integration Android alert dismiss falls back to Back without a dismiss button', async () => { await withProviderScenarioResource( async () => await createAndroidSettingsWorld({ snapshotXml: androidButtonlessAlertXml }), diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 63fabae08..67aa25f6b 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -49,6 +49,8 @@ export async function createAndroidSettingsWorld(options?: { nativeTextInjection?: boolean; nativeTouchInjection?: boolean; snapshotXml?: () => string; + dumpsysWindow?: () => string; + onAdbExec?: (args: string[]) => void; }): Promise { const hostAdbGuard = installFakeHostAdbGuard(); const adbCalls: string[][] = []; @@ -77,13 +79,17 @@ export async function createAndroidSettingsWorld(options?: { const adbProvider: AndroidAdbProvider = { exec: async (args) => { adbCalls.push([...args]); + options?.onAdbExec?.([...args]); if (args[0] === 'shell' && args[1] === 'input' && args[2] === 'text') { searchText = String(args[3] ?? '').replaceAll('%s', ' '); } if (args.join(' ') === 'shell cmd clipboard set text android otp') { clipboardText = 'android otp'; } - return androidAdbResult(args, searchText, clipboardText, options?.snapshotXml); + return androidAdbResult(args, searchText, clipboardText, { + snapshotXml: options?.snapshotXml, + dumpsysWindow: options?.dumpsysWindow, + }); }, install: async (apk, options) => { apkInstallCalls.push({ apkPath: apk, replace: options?.replace }); @@ -190,14 +196,21 @@ function androidAdbResult( args: string[], searchText: string, clipboardText: string, - snapshotXml?: () => string, + options: { + snapshotXml?: () => string; + dumpsysWindow?: () => string; + }, ): { stdout: string; stderr: string; exitCode: number; stdoutBuffer?: Buffer } { const key = args.join(' '); return ( androidDeviceStateAdbResult(key, clipboardText) ?? androidMetricsAdbResult(key) ?? - androidPackageAdbResult(key, args) ?? - androidCaptureAdbResult(key, searchText, snapshotXml) ?? { stdout: '', stderr: '', exitCode: 0 } + androidPackageAdbResult(key, args, options.dumpsysWindow) ?? + androidCaptureAdbResult(key, searchText, options.snapshotXml) ?? { + stdout: '', + stderr: '', + exitCode: 0, + } ); } @@ -270,7 +283,11 @@ function androidMetricsAdbResult(key: string): AndroidAdbResult | undefined { return undefined; } -function androidPackageAdbResult(key: string, args: string[]): AndroidAdbResult | undefined { +function androidPackageAdbResult( + key: string, + args: string[], + dumpsysWindow?: () => string, +): AndroidAdbResult | undefined { if ( args.slice(0, 7).join(' ') === 'shell cmd package query-activities --brief -a android.intent.action.MAIN' @@ -288,9 +305,9 @@ function androidPackageAdbResult(key: string, args: string[]): AndroidAdbResult exitCode: 0, }; } - if (key === 'shell dumpsys window windows') { + if (key === 'shell dumpsys window windows' || key === 'shell dumpsys window') { return { - stdout: 'mCurrentFocus=Window{42 u0 com.android.settings/.Settings}\n', + stdout: dumpsysWindow?.() ?? 'mCurrentFocus=Window{42 u0 com.android.settings/.Settings}\n', stderr: '', exitCode: 0, };