Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/command-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,26 @@ export const DAEMON_COMMAND_GROUPS = {
PUBLIC_COMMANDS.type,
PUBLIC_COMMANDS.wait,
),
androidBlockingDialogGuardedAction: commandSet(
PUBLIC_COMMANDS.back,
PUBLIC_COMMANDS.click,
PUBLIC_COMMANDS.fill,
PUBLIC_COMMANDS.focus,
PUBLIC_COMMANDS.gesture,
PUBLIC_COMMANDS.home,
PUBLIC_COMMANDS.keyboard,
PUBLIC_COMMANDS.longPress,
'fling',
'pan',
'pinch',
PUBLIC_COMMANDS.press,
PUBLIC_COMMANDS.rotate,
'rotate-gesture',
PUBLIC_COMMANDS.scroll,
PUBLIC_COMMANDS.swipe,
'transform-gesture',
PUBLIC_COMMANDS.type,
),
selectorValidationExempt: commandSet(
INTERNAL_COMMANDS.sessionList,
PUBLIC_COMMANDS.devices,
Expand Down
1 change: 1 addition & 0 deletions src/daemon/__tests__/request-router-android-modal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => {
...actual,
openAndroidApp: vi.fn(async () => {}),
getAndroidAppState: vi.fn(async () => ({ package: 'com.android.settings' })),
getAndroidBlockingDialogFocus: vi.fn(async () => null),
};
});

Expand Down
8 changes: 8 additions & 0 deletions src/daemon/__tests__/request-router-screenshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ vi.mock('../../core/dispatch.ts', async (importOriginal) => {
return { ...actual, dispatchCommand: vi.fn(async () => ({})) };
});

vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../platforms/android/app-lifecycle.ts')>();
return {
...actual,
getAndroidBlockingDialogFocus: vi.fn(async () => null),
};
});

import { dispatchCommand } from '../../core/dispatch.ts';
import { createRequestHandler } from '../request-router.ts';
import { dispatchScreenshotViaRuntime } from '../screenshot-runtime.ts';
Expand Down
222 changes: 201 additions & 21 deletions src/daemon/android-system-dialog.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,37 @@
import { getAndroidAppState, openAndroidApp } from '../platforms/android/app-lifecycle.ts';
import {
getAndroidAppState,
getAndroidBlockingDialogFocus,
openAndroidApp,
type AndroidBlockingDialogFocus,
} from '../platforms/android/app-lifecycle.ts';
import { snapshotAndroid } from '../platforms/android/snapshot.ts';
import { runAndroidAdb } from '../platforms/android/adb.ts';
import { emitDiagnostic } from '../utils/diagnostics.ts';
import { AppError } from '../utils/errors.ts';
import { centerOfRect, attachRefs, type SnapshotNode } from '../utils/snapshot.ts';
import { sleep } from '../utils/timeouts.ts';
import { pruneGroupNodes } from './snapshot-processing.ts';
import type { SessionState } from './types.ts';

const ANDROID_BLOCKING_MODAL_PATTERN = /\bis(?:n't| not)\s+responding\b/i;
const ANDROID_BLOCKING_MODAL_PATTERN = /\bis(?:n(?:'|&apos;|&#39;)?t| not)\s+responding\b/i;
const ANDROID_CLOSE_APP_PATTERN = /^close app$/i;
const ANDROID_MODAL_POLL_MS = 500;
const ANDROID_MODAL_POLL_ATTEMPTS = 12;
const ANDROID_BLOCKING_DIALOG_HINT =
'Wait for Android to recover, close the dialog, restart the app, or reboot the emulator, then retry.';

export type AndroidBlockingDialogRecoveryResult = 'absent' | 'recovered' | 'failed';
export type AndroidBlockingDialogReadinessResult =
| { status: 'clear' }
| { status: 'recovered'; warning: string };
type AndroidDialogButtonTapResult =
| { ok: true; x: number; y: number }
| {
ok: false;
exitCode: number;
stdout: string;
stderr: string;
};

export async function recoverAndroidBlockingSystemDialog(params: {
session: SessionState;
Expand All @@ -30,13 +49,8 @@ export async function recoverAndroidBlockingSystemDialog(params: {
return 'absent';
}

const { x, y } = centerOfRect(closeAppButton.rect);
const tapResult = await runAndroidAdb(
session.device,
['shell', 'input', 'tap', String(Math.round(x)), String(Math.round(y))],
{ allowFailure: true },
);
if (tapResult.exitCode !== 0) {
const tapResult = await tapAndroidDialogButton(session, closeAppButton);
if (!tapResult.ok) {
emitDiagnostic({
level: 'warn',
phase: 'android_blocking_dialog_tap_failed',
Expand Down Expand Up @@ -66,7 +80,7 @@ export async function recoverAndroidBlockingSystemDialog(params: {

if (session.appBundleId) {
await openAndroidApp(session.device, session.appBundleId);
const focused = await waitForFocusedAndroidApp(session, session.appBundleId);
const focused = await waitForAndroidAppFocus(session, session.appBundleId);
if (!focused) {
emitDiagnostic({
level: 'warn',
Expand All @@ -88,8 +102,8 @@ export async function recoverAndroidBlockingSystemDialog(params: {
session: session.name,
deviceId: session.device.id,
appBundleId: session.appBundleId,
x,
y,
x: tapResult.x,
y: tapResult.y,
},
});
return 'recovered';
Expand All @@ -107,6 +121,126 @@ export async function recoverAndroidBlockingSystemDialog(params: {
}
}

export async function ensureAndroidBlockingSystemDialogReady(params: {
session: SessionState;
command: string;
phase: 'before-command' | 'after-command';
}): Promise<AndroidBlockingDialogReadinessResult> {
const { session, command } = params;
if (session.device.platform !== 'android') return { status: 'clear' };

const focus = await getAndroidBlockingDialogFocus(session.device);
if (!focus) return { status: 'clear' };

if (isSessionAppAnr(session, focus)) {
const recovered = await recoverAppOwnedAndroidBlockingSystemDialogSafely(session);
if (recovered) {
const warning = `Recovered Android app ANR before ${command}: closed and relaunched ${session.appBundleId}.`;
if (params.phase === 'before-command') return { status: 'recovered', warning };

throw androidBlockingDialogError({
session,
command,
focus,
message: `Android app ANR appeared after ${command}; ${session.appBundleId} was closed and relaunched. Retry the command against the fresh app session.`,
hint: 'Retry the command. If the ANR returns, inspect app logs or restart the emulator.',
});
}

throw androidBlockingDialogError({
session,
command,
focus,
message: `Android app ANR blocked ${command}: ${formatAndroidBlockingDialogFocus(focus)}. Automatic recovery failed.`,
hint: ANDROID_BLOCKING_DIALOG_HINT,
});
}

throw androidBlockingDialogError({
session,
command,
focus,
message: `Android system dialog is blocking ${command}: ${formatAndroidBlockingDialogFocus(focus)}.`,
hint: ANDROID_BLOCKING_DIALOG_HINT,
});
}

async function recoverAppOwnedAndroidBlockingSystemDialogSafely(
session: SessionState,
): Promise<boolean> {
try {
return await recoverAppOwnedAndroidBlockingSystemDialog(session);
} catch (error) {
emitDiagnostic({
level: 'warn',
phase: 'android_app_anr_recovery_failed',
data: {
session: session.name,
deviceId: session.device.id,
appBundleId: session.appBundleId,
error: error instanceof Error ? error.message : String(error),
},
});
return false;
}
}

function isSessionAppAnr(session: SessionState, focus: AndroidBlockingDialogFocus): boolean {
return Boolean(session.appBundleId && focus.package === session.appBundleId);
}

async function recoverAppOwnedAndroidBlockingSystemDialog(session: SessionState): Promise<boolean> {
if (!session.appBundleId) return false;

const nodes = await readAndroidSnapshotNodes(session);
const closeAppButton = findCloseAppButton(nodes, { requireDialogSignal: false });
if (!closeAppButton?.rect) return false;

const tapResult = await tapAndroidDialogButton(session, closeAppButton);
if (!tapResult.ok) return false;

await openAndroidApp(session.device, session.appBundleId);
const focused = await waitForAndroidAppFocus(session, session.appBundleId, {
requireNoBlockingDialog: true,
});
if (focused) {
emitDiagnostic({
level: 'warn',
phase: 'android_app_anr_recovered',
data: {
session: session.name,
deviceId: session.device.id,
appBundleId: session.appBundleId,
x: tapResult.x,
y: tapResult.y,
},
});
}
return focused;
}

function androidBlockingDialogError(params: {
session: SessionState;
command: string;
focus: AndroidBlockingDialogFocus;
message: string;
hint: string;
}): AppError {
const { session, command, focus, message, hint } = params;
return new AppError('COMMAND_FAILED', message, {
command,
expectedPackage: session.appBundleId,
focusedPackage: focus.package,
focusedWindow: focus.focusedWindow,
rawFocus: focus.raw,
hint,
});
}

function formatAndroidBlockingDialogFocus(focus: AndroidBlockingDialogFocus): string {
return focus.package ? `${focus.focusedWindow} (package ${focus.package})` : focus.focusedWindow;
}

async function readAndroidSnapshotNodes(session: SessionState): Promise<SnapshotNode[]> {
const rawSnapshot = await snapshotAndroid(session.device, {
interactiveOnly: false,
Expand All @@ -115,13 +249,41 @@ async function readAndroidSnapshotNodes(session: SessionState): Promise<Snapshot
return attachRefs(pruneGroupNodes(rawSnapshot.nodes));
}

function findCloseAppButton(nodes: SnapshotNode[]): SnapshotNode | undefined {
if (!containsBlockingDialog(nodes)) {
async function tapAndroidDialogButton(
session: SessionState,
button: SnapshotNode,
): Promise<AndroidDialogButtonTapResult> {
if (!button.rect) {
return { ok: false, exitCode: 1, stdout: '', stderr: 'button has no rect' };
}
const { x, y } = centerOfRect(button.rect);
const result = await runAndroidAdb(
session.device,
['shell', 'input', 'tap', String(Math.round(x)), String(Math.round(y))],
{ allowFailure: true },
);
if (result.exitCode !== 0) {
return {
ok: false,
exitCode: result.exitCode,
stdout: result.stdout.trim(),
stderr: result.stderr.trim(),
};
}
return { ok: true, x, y };
}

function findCloseAppButton(
nodes: SnapshotNode[],
options: { requireDialogSignal?: boolean } = {},
): SnapshotNode | undefined {
if (options.requireDialogSignal !== false && !containsBlockingDialog(nodes)) {
return undefined;
}
return nodes.find((node) => {
const text = readNodeText(node);
return text.length > 0 && ANDROID_CLOSE_APP_PATTERN.test(text) && node.rect;
return (
readNodeTextParts(node).some((text) => ANDROID_CLOSE_APP_PATTERN.test(text)) && node.rect
);
});
}

Expand All @@ -137,17 +299,28 @@ async function waitForBlockingDialogToDismiss(session: SessionState): Promise<bo
return !containsBlockingDialog(nodes);
}

async function waitForFocusedAndroidApp(
async function waitForAndroidAppFocus(
session: SessionState,
appBundleId: string,
options: { requireNoBlockingDialog?: boolean } = {},
): Promise<boolean> {
for (let attempt = 0; attempt < ANDROID_MODAL_POLL_ATTEMPTS; attempt += 1) {
const state = await getAndroidAppState(session.device);
if (state.package === appBundleId) {
if (await isAndroidAppFocused(session, appBundleId, options)) {
return true;
}
await sleep(ANDROID_MODAL_POLL_MS);
}
return await isAndroidAppFocused(session, appBundleId, options);
}

async function isAndroidAppFocused(
session: SessionState,
appBundleId: string,
options: { requireNoBlockingDialog?: boolean },
): Promise<boolean> {
if (options.requireNoBlockingDialog && (await getAndroidBlockingDialogFocus(session.device))) {
return false;
}
const state = await getAndroidAppState(session.device);
return state.package === appBundleId;
}
Expand All @@ -157,14 +330,21 @@ function readNodeText(node: {
value?: string | number | boolean | null;
identifier?: string;
}): string {
return readNodeTextParts(node).join(' ').trim();
}

function readNodeTextParts(node: {
label?: string;
value?: string | number | boolean | null;
identifier?: string;
}): string[] {
const parts = [node.label, node.identifier];
if (typeof node.value === 'string' && node.value.trim().length > 0) {
parts.push(node.value);
}
return parts
.filter((part): part is string => typeof part === 'string' && part.trim().length > 0)
.join(' ')
.trim();
.map((part) => part.trim());
}

function containsBlockingDialog(nodes: SnapshotNode[]): boolean {
Expand Down
9 changes: 8 additions & 1 deletion src/daemon/handlers/__tests__/interaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ vi.mock('../../../platforms/android/app-lifecycle.ts', async (importOriginal) =>
return {
...actual,
getAndroidAppState: vi.fn(async () => ({})),
getAndroidBlockingDialogFocus: vi.fn(async () => null),
};
});

Expand All @@ -64,11 +65,15 @@ vi.mock('../../../platforms/ios/runner-client.ts', async (importOriginal) => {
});

import { dispatchCommand } from '../../../core/dispatch.ts';
import { getAndroidAppState } from '../../../platforms/android/app-lifecycle.ts';
import {
getAndroidAppState,
getAndroidBlockingDialogFocus,
} from '../../../platforms/android/app-lifecycle.ts';
import { getAndroidScreenSize } from '../../../platforms/android/input-actions.ts';
import { captureSnapshotForSession } from '../interaction-snapshot.ts';
const mockDispatch = vi.mocked(dispatchCommand);
const mockGetAndroidAppState = vi.mocked(getAndroidAppState);
const mockGetAndroidBlockingDialogFocus = vi.mocked(getAndroidBlockingDialogFocus);
const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize);
const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotForSession);

Expand Down Expand Up @@ -132,6 +137,8 @@ beforeEach(() => {
mockDispatch.mockResolvedValue({});
mockGetAndroidAppState.mockReset();
mockGetAndroidAppState.mockResolvedValue({});
mockGetAndroidBlockingDialogFocus.mockReset();
mockGetAndroidBlockingDialogFocus.mockResolvedValue(null);
mockGetAndroidScreenSize.mockReset();
mockGetAndroidScreenSize.mockResolvedValue({ width: 1344, height: 2992 });
mockCaptureSnapshotForSession.mockReset();
Expand Down
Loading
Loading