Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 210 additions & 0 deletions src/daemon/android-snapshot-timeout-evidence.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
import { promises as fs } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { DaemonResponse, SessionState } from './types.ts';
import { dispatchCommand } from '../core/dispatch.ts';
import { emitDiagnostic } from '../utils/diagnostics.ts';
import { normalizeError, type NormalizedError } from '../utils/errors.ts';
import type { ScreenshotOverlayRef } from '../utils/snapshot.ts';
import { contextFromFlags } from './context.ts';
import { annotateScreenshotWithRefs } from './screenshot-overlay.ts';

type CapturedAndroidSnapshotTimeoutEvidenceBase = {
path: string;
overlayRefsRequested: true;
};

type AndroidSnapshotTimeoutEvidence =
| {
captureFailed: true;
error: string;
}
| (CapturedAndroidSnapshotTimeoutEvidenceBase & {
overlayRefSource: 'unavailable';
overlayRefsAnnotated: false;
overlayRefCount: 0;
})
| (CapturedAndroidSnapshotTimeoutEvidenceBase & {
overlayRefSource: 'session-snapshot';
overlayRefsAnnotated: boolean;
overlayRefCount: number;
overlayRefs: ScreenshotOverlayRef[];
})
| (CapturedAndroidSnapshotTimeoutEvidenceBase & {
overlayRefSource: 'session-snapshot';
overlayRefsAnnotated: false;
overlayRefCount: 0;
overlayAnnotationError: string;
});

export async function maybeBuildAndroidSnapshotTimeoutFailure(params: {
error: unknown;
command: 'snapshot' | 'diff';
logPath: string;
session: SessionState | undefined;
device: SessionState['device'];
}): Promise<Extract<DaemonResponse, { ok: false }> | undefined> {
if (params.command !== 'snapshot') return undefined;
if (params.device.platform !== 'android') return undefined;

const normalized = normalizeError(params.error);
if (!isAndroidSnapshotTimeoutError(normalized)) return undefined;

return {
ok: false,
error: {
...normalized,
details: {
...(normalized.details ?? {}),
androidSnapshotTimeoutScreenshot: await captureAndroidSnapshotTimeoutEvidence(params),
},
},
};
}

async function captureAndroidSnapshotTimeoutEvidence(params: {
logPath: string;
session: SessionState | undefined;
device: SessionState['device'];
}): Promise<AndroidSnapshotTimeoutEvidence> {
try {
const tempDir = await fs.mkdtemp(
path.join(os.tmpdir(), 'agent-device-android-snapshot-timeout-'),
);
const screenshotPath = path.join(tempDir, 'snapshot-timeout-overlay-refs.png');
const data = await dispatchCommand(params.device, 'screenshot', [screenshotPath], undefined, {
...contextFromFlags(
params.logPath,
// Use a fresh unstabilized screenshot context; inheriting snapshot flags could repeat the
// accessibility stabilization timeout that this fallback is trying to avoid.
{ screenshotNoStabilize: true },
params.session?.appBundleId,
params.session?.trace?.outPath,
),
surface: params.session?.surface,
});
const resolvedPath = resolveCapturedScreenshotPath(data, screenshotPath);
await fs.access(resolvedPath);
const evidence = await annotateAndroidSnapshotTimeoutEvidence(resolvedPath, params.session);

emitDiagnostic({
level: 'warn',
phase: 'android_snapshot_timeout_screenshot_captured',
data: {
path: resolvedPath,
overlayRefCount: 'overlayRefCount' in evidence ? evidence.overlayRefCount : undefined,
overlayRefsAnnotated:
'overlayRefsAnnotated' in evidence ? evidence.overlayRefsAnnotated : undefined,
},
});
return evidence;
} catch (error) {
const normalized = normalizeError(error);
emitDiagnostic({
level: 'warn',
phase: 'android_snapshot_timeout_screenshot_failed',
data: { error: normalized.message },
});
return {
captureFailed: true,
error: normalized.message,
};
}
}

async function annotateAndroidSnapshotTimeoutEvidence(
screenshotPath: string,
session: SessionState | undefined,
): Promise<AndroidSnapshotTimeoutEvidence> {
if (!session?.snapshot) {
return {
path: screenshotPath,
overlayRefsRequested: true,
overlayRefsAnnotated: false,
overlayRefSource: 'unavailable',
overlayRefCount: 0,
};
}

try {
const overlayRefs = await annotateScreenshotWithRefs({
screenshotPath,
snapshot: session.snapshot,
});
return {
path: screenshotPath,
overlayRefsRequested: true,
overlayRefsAnnotated: overlayRefs.length > 0,
overlayRefCount: overlayRefs.length,
overlayRefSource: 'session-snapshot',
overlayRefs,
};
} catch (error) {
const normalized = normalizeError(error);
emitDiagnostic({
level: 'warn',
phase: 'android_snapshot_timeout_screenshot_overlay_failed',
data: { path: screenshotPath, error: normalized.message },
});
return {
path: screenshotPath,
overlayRefsRequested: true,
overlayRefsAnnotated: false,
overlayRefSource: 'session-snapshot',
overlayRefCount: 0,
overlayAnnotationError: normalized.message,
};
}
}

function resolveCapturedScreenshotPath(data: unknown, fallbackPath: string): string {
return hasStringPath(data) ? data.path : fallbackPath;
}

function hasStringPath(value: unknown): value is { path: string } {
return (
typeof value === 'object' && value !== null && 'path' in value && typeof value.path === 'string'
);
}

function isAndroidSnapshotTimeoutError(error: NormalizedError): boolean {
if (error.code !== 'COMMAND_FAILED') return false;
return (
hasKnownAndroidSnapshotTimeoutMessage(error) ||
hasHelperTimeoutDetails(error.details?.helper) ||
hasUiAutomatorDumpTimeoutDetails(error.details)
);
}

function hasKnownAndroidSnapshotTimeoutMessage(error: NormalizedError): boolean {
const text = `${error.message}\n${error.hint ?? ''}`;
if (/Android UI hierarchy dump timed out/i.test(text)) return true;
if (/Stock UIAutomator fallback was skipped/i.test(text)) return true;
return /Android accessibility snapshots can be blocked/i.test(text);
}

function hasHelperTimeoutDetails(helper: unknown): boolean {
if (!helper || typeof helper !== 'object') return false;
const helperRecord = helper as Record<string, unknown>;
const errorType = String(helperRecord.errorType ?? '');
const message = String(helperRecord.message ?? '');
return /TimeoutException/i.test(errorType) || /timed out/i.test(message);
}

function hasUiAutomatorDumpTimeoutDetails(details: Record<string, unknown> | undefined): boolean {
if (!details) return false;
const timeoutMs = details?.timeoutMs;
const cmd = details?.cmd;
const args = normalizeArgList(details?.args);
return (
typeof timeoutMs === 'number' &&
cmd === 'adb' &&
args.includes('uiautomator') &&
args.includes('dump')
);
}

function normalizeArgList(rawArgs: unknown): string[] {
if (Array.isArray(rawArgs)) return rawArgs.map(String);
return typeof rawArgs === 'string' ? rawArgs.split(/\s+/) : [];
}
102 changes: 102 additions & 0 deletions src/daemon/handlers/__tests__/snapshot-handler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { test, expect, vi, beforeEach } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { PNG } from 'pngjs';
import { handleSnapshotCommands } from '../snapshot.ts';
import { captureSnapshot } from '../snapshot-capture.ts';
import { SessionStore } from '../../session-store.ts';
Expand Down Expand Up @@ -80,6 +81,87 @@ beforeEach(() => {
mockRunnerCommand.mockResolvedValue({});
});

function writeSolidPng(filePath: string, width = 390, height = 844): void {
const png = new PNG({ width, height });
for (let index = 0; index < png.data.length; index += 4) {
png.data[index] = 255;
png.data[index + 1] = 255;
png.data[index + 2] = 255;
png.data[index + 3] = 255;
}
fs.writeFileSync(filePath, PNG.sync.write(png));
}

function makeAndroidTimeoutEvidenceSession(sessionName: string): SessionStore {
const sessionStore = makeSessionStore();
const session = makeSession(sessionName, androidDevice);
session.snapshot = {
nodes: [
{
ref: 'e1',
index: 0,
depth: 0,
type: 'android.widget.Button',
label: 'Continue',
hittable: true,
rect: { x: 20, y: 40, width: 120, height: 48 },
},
],
createdAt: Date.now(),
backend: 'android',
};
sessionStore.set(sessionName, session);
return sessionStore;
}

function mockAndroidTimeoutEvidenceDispatch(): void {
mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => {
if (command === 'snapshot') throw androidSnapshotTimeoutError();
if (command === 'screenshot') {
const screenshotPath = positionals[0];
expect(context?.screenshotNoStabilize).toBe(true);
writeSolidPng(screenshotPath);
return { path: screenshotPath };
}
return {};
});
}

function androidSnapshotTimeoutError(): AppError {
return new AppError(
'COMMAND_FAILED',
'Android UI hierarchy dump timed out while waiting for the UI to become idle.',
{
cmd: 'adb',
args: ['exec-out', 'uiautomator', 'dump', '/dev/tty'],
timeoutMs: 8000,
hint: 'Android accessibility snapshots can be blocked by busy or continuously changing app UI. Use screenshot as visual truth after this timeout.',
},
);
}

function expectAndroidTimeoutEvidence(
response: Awaited<ReturnType<typeof handleSnapshotCommands>>,
) {
if (!response) throw new Error('Expected snapshot response');
if (response.ok) throw new Error('Expected snapshot timeout failure');
expect(response.error.message).toMatch(/UI hierarchy dump timed out/i);
expect(response.error.hint).toMatch(/Use screenshot as visual truth/i);
assertAndroidTimeoutEvidencePayload(response.error.details?.androidSnapshotTimeoutScreenshot);
}

function assertAndroidTimeoutEvidencePayload(evidence: unknown) {
if (!evidence || typeof evidence !== 'object') {
throw new Error('Expected Android snapshot timeout screenshot evidence');
}
const record = evidence as Record<string, unknown>;
expect(record.path).toEqual(expect.stringContaining('snapshot-timeout-overlay-refs.png'));
expect(fs.existsSync(record.path as string)).toBe(true);
expect(record.overlayRefsAnnotated).toBe(true);
expect(record.overlayRefCount).toBe(1);
expect(record.overlayRefs).toEqual([expect.objectContaining({ ref: 'e1', label: 'Continue' })]);
}

async function runWaitCommand(
sessionName: string,
device: SessionState['device'],
Expand Down Expand Up @@ -268,6 +350,26 @@ test('snapshot surfaces filtered-to-zero Android guidance for interactive snapsh
}
});

test('snapshot timeout captures Android screenshot evidence with overlay refs', async () => {
const sessionName = 'android-timeout-evidence';
const sessionStore = makeAndroidTimeoutEvidenceSession(sessionName);
mockAndroidTimeoutEvidenceDispatch();
const response = await handleSnapshotCommands({
req: {
token: 't',
session: sessionName,
command: 'snapshot',
positionals: [],
flags: {},
},
sessionName,
logPath: '/tmp/daemon.log',
sessionStore,
});
expectAndroidTimeoutEvidence(response);
expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['snapshot', 'screenshot']);
});

test('snapshot warns when recent snapshot node count collapses sharply', async () => {
const sessionStore = makeSessionStore();
const sessionName = 'android-stale-collapse';
Expand Down
26 changes: 20 additions & 6 deletions src/daemon/snapshot-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
} from './handlers/snapshot-session.ts';
import { createDaemonRuntimePolicy } from './runtime-policy.ts';
import { createDaemonRuntimeSessionStore } from './runtime-session.ts';
import { maybeBuildAndroidSnapshotTimeoutFailure } from './android-snapshot-timeout-evidence.ts';

export async function dispatchSnapshotViaRuntime(params: {
req: DaemonRequest;
Expand Down Expand Up @@ -125,12 +126,25 @@ async function dispatchSnapshotRuntimeCommand(
device,
snapshotScope: resolvedScope.scope,
});
const result = await params.execute({
runtime,
sessionName,
req,
snapshotScope: resolvedScope.scope,
});
let result: Awaited<ReturnType<SnapshotRuntimeCommandParams['execute']>>;
try {
result = await params.execute({
runtime,
sessionName,
req,
snapshotScope: resolvedScope.scope,
});
} catch (error) {
const timeoutResponse = await maybeBuildAndroidSnapshotTimeoutFailure({
error,
command: params.command,
logPath,
session,
device,
});
if (!timeoutResponse) throw error;
return timeoutResponse;
}
recordSnapshotRuntimeAction({
req,
sessionName,
Expand Down
Loading