Skip to content

Commit 87f087c

Browse files
thymikeethymikee
authored
fix: capture Android snapshot timeout evidence (#591)
* fix: capture Android snapshot timeout evidence * refactor: isolate Android snapshot timeout evidence * fix: satisfy fallow snapshot timeout checks * refactor: tighten snapshot timeout evidence payload * refactor: tighten Android timeout evidence cleanup
1 parent ea21793 commit 87f087c

3 files changed

Lines changed: 332 additions & 6 deletions

File tree

‎src/daemon/android-snapshot-timeout-evidence.ts‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
import { promises as fs } from 'node:fs';
2+
import os from 'node:os';
3+
import path from 'node:path';
4+
import type { DaemonResponse, SessionState } from './types.ts';
5+
import { dispatchCommand } from '../core/dispatch.ts';
6+
import { emitDiagnostic } from '../utils/diagnostics.ts';
7+
import { normalizeError, type NormalizedError } from '../utils/errors.ts';
8+
import type { ScreenshotOverlayRef } from '../utils/snapshot.ts';
9+
import { contextFromFlags } from './context.ts';
10+
import { annotateScreenshotWithRefs } from './screenshot-overlay.ts';
11+
12+
type CapturedAndroidSnapshotTimeoutEvidenceBase = {
13+
path: string;
14+
overlayRefsRequested: true;
15+
};
16+
17+
type AndroidSnapshotTimeoutEvidence =
18+
| {
19+
captureFailed: true;
20+
error: string;
21+
}
22+
| (CapturedAndroidSnapshotTimeoutEvidenceBase & {
23+
overlayRefSource: 'unavailable';
24+
overlayRefsAnnotated: false;
25+
overlayRefCount: 0;
26+
})
27+
| (CapturedAndroidSnapshotTimeoutEvidenceBase & {
28+
overlayRefSource: 'session-snapshot';
29+
overlayRefsAnnotated: boolean;
30+
overlayRefCount: number;
31+
overlayRefs: ScreenshotOverlayRef[];
32+
})
33+
| (CapturedAndroidSnapshotTimeoutEvidenceBase & {
34+
overlayRefSource: 'session-snapshot';
35+
overlayRefsAnnotated: false;
36+
overlayRefCount: 0;
37+
overlayAnnotationError: string;
38+
});
39+
40+
export async function maybeBuildAndroidSnapshotTimeoutFailure(params: {
41+
error: unknown;
42+
command: 'snapshot' | 'diff';
43+
logPath: string;
44+
session: SessionState | undefined;
45+
device: SessionState['device'];
46+
}): Promise<Extract<DaemonResponse, { ok: false }> | undefined> {
47+
if (params.command !== 'snapshot') return undefined;
48+
if (params.device.platform !== 'android') return undefined;
49+
50+
const normalized = normalizeError(params.error);
51+
if (!isAndroidSnapshotTimeoutError(normalized)) return undefined;
52+
53+
return {
54+
ok: false,
55+
error: {
56+
...normalized,
57+
details: {
58+
...(normalized.details ?? {}),
59+
androidSnapshotTimeoutScreenshot: await captureAndroidSnapshotTimeoutEvidence(params),
60+
},
61+
},
62+
};
63+
}
64+
65+
async function captureAndroidSnapshotTimeoutEvidence(params: {
66+
logPath: string;
67+
session: SessionState | undefined;
68+
device: SessionState['device'];
69+
}): Promise<AndroidSnapshotTimeoutEvidence> {
70+
try {
71+
const tempDir = await fs.mkdtemp(
72+
path.join(os.tmpdir(), 'agent-device-android-snapshot-timeout-'),
73+
);
74+
const screenshotPath = path.join(tempDir, 'snapshot-timeout-overlay-refs.png');
75+
const data = await dispatchCommand(params.device, 'screenshot', [screenshotPath], undefined, {
76+
...contextFromFlags(
77+
params.logPath,
78+
// Use a fresh unstabilized screenshot context; inheriting snapshot flags could repeat the
79+
// accessibility stabilization timeout that this fallback is trying to avoid.
80+
{ screenshotNoStabilize: true },
81+
params.session?.appBundleId,
82+
params.session?.trace?.outPath,
83+
),
84+
surface: params.session?.surface,
85+
});
86+
const resolvedPath = resolveCapturedScreenshotPath(data, screenshotPath);
87+
await fs.access(resolvedPath);
88+
const evidence = await annotateAndroidSnapshotTimeoutEvidence(resolvedPath, params.session);
89+
90+
emitDiagnostic({
91+
level: 'warn',
92+
phase: 'android_snapshot_timeout_screenshot_captured',
93+
data: {
94+
path: resolvedPath,
95+
overlayRefCount: 'overlayRefCount' in evidence ? evidence.overlayRefCount : undefined,
96+
overlayRefsAnnotated:
97+
'overlayRefsAnnotated' in evidence ? evidence.overlayRefsAnnotated : undefined,
98+
},
99+
});
100+
return evidence;
101+
} catch (error) {
102+
const normalized = normalizeError(error);
103+
emitDiagnostic({
104+
level: 'warn',
105+
phase: 'android_snapshot_timeout_screenshot_failed',
106+
data: { error: normalized.message },
107+
});
108+
return {
109+
captureFailed: true,
110+
error: normalized.message,
111+
};
112+
}
113+
}
114+
115+
async function annotateAndroidSnapshotTimeoutEvidence(
116+
screenshotPath: string,
117+
session: SessionState | undefined,
118+
): Promise<AndroidSnapshotTimeoutEvidence> {
119+
if (!session?.snapshot) {
120+
return {
121+
path: screenshotPath,
122+
overlayRefsRequested: true,
123+
overlayRefsAnnotated: false,
124+
overlayRefSource: 'unavailable',
125+
overlayRefCount: 0,
126+
};
127+
}
128+
129+
try {
130+
const overlayRefs = await annotateScreenshotWithRefs({
131+
screenshotPath,
132+
snapshot: session.snapshot,
133+
});
134+
return {
135+
path: screenshotPath,
136+
overlayRefsRequested: true,
137+
overlayRefsAnnotated: overlayRefs.length > 0,
138+
overlayRefCount: overlayRefs.length,
139+
overlayRefSource: 'session-snapshot',
140+
overlayRefs,
141+
};
142+
} catch (error) {
143+
const normalized = normalizeError(error);
144+
emitDiagnostic({
145+
level: 'warn',
146+
phase: 'android_snapshot_timeout_screenshot_overlay_failed',
147+
data: { path: screenshotPath, error: normalized.message },
148+
});
149+
return {
150+
path: screenshotPath,
151+
overlayRefsRequested: true,
152+
overlayRefsAnnotated: false,
153+
overlayRefSource: 'session-snapshot',
154+
overlayRefCount: 0,
155+
overlayAnnotationError: normalized.message,
156+
};
157+
}
158+
}
159+
160+
function resolveCapturedScreenshotPath(data: unknown, fallbackPath: string): string {
161+
return hasStringPath(data) ? data.path : fallbackPath;
162+
}
163+
164+
function hasStringPath(value: unknown): value is { path: string } {
165+
return (
166+
typeof value === 'object' && value !== null && 'path' in value && typeof value.path === 'string'
167+
);
168+
}
169+
170+
function isAndroidSnapshotTimeoutError(error: NormalizedError): boolean {
171+
if (error.code !== 'COMMAND_FAILED') return false;
172+
return (
173+
hasKnownAndroidSnapshotTimeoutMessage(error) ||
174+
hasHelperTimeoutDetails(error.details?.helper) ||
175+
hasUiAutomatorDumpTimeoutDetails(error.details)
176+
);
177+
}
178+
179+
function hasKnownAndroidSnapshotTimeoutMessage(error: NormalizedError): boolean {
180+
const text = `${error.message}\n${error.hint ?? ''}`;
181+
if (/Android UI hierarchy dump timed out/i.test(text)) return true;
182+
if (/Stock UIAutomator fallback was skipped/i.test(text)) return true;
183+
return /Android accessibility snapshots can be blocked/i.test(text);
184+
}
185+
186+
function hasHelperTimeoutDetails(helper: unknown): boolean {
187+
if (!helper || typeof helper !== 'object') return false;
188+
const helperRecord = helper as Record<string, unknown>;
189+
const errorType = String(helperRecord.errorType ?? '');
190+
const message = String(helperRecord.message ?? '');
191+
return /TimeoutException/i.test(errorType) || /timed out/i.test(message);
192+
}
193+
194+
function hasUiAutomatorDumpTimeoutDetails(details: Record<string, unknown> | undefined): boolean {
195+
if (!details) return false;
196+
const timeoutMs = details?.timeoutMs;
197+
const cmd = details?.cmd;
198+
const args = normalizeArgList(details?.args);
199+
return (
200+
typeof timeoutMs === 'number' &&
201+
cmd === 'adb' &&
202+
args.includes('uiautomator') &&
203+
args.includes('dump')
204+
);
205+
}
206+
207+
function normalizeArgList(rawArgs: unknown): string[] {
208+
if (Array.isArray(rawArgs)) return rawArgs.map(String);
209+
return typeof rawArgs === 'string' ? rawArgs.split(/\s+/) : [];
210+
}

‎src/daemon/handlers/__tests__/snapshot-handler.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { test, expect, vi, beforeEach } from 'vitest';
22
import fs from 'node:fs';
33
import os from 'node:os';
44
import path from 'node:path';
5+
import { PNG } from 'pngjs';
56
import { handleSnapshotCommands } from '../snapshot.ts';
67
import { captureSnapshot } from '../snapshot-capture.ts';
78
import { SessionStore } from '../../session-store.ts';
@@ -80,6 +81,87 @@ beforeEach(() => {
8081
mockRunnerCommand.mockResolvedValue({});
8182
});
8283

84+
function writeSolidPng(filePath: string, width = 390, height = 844): void {
85+
const png = new PNG({ width, height });
86+
for (let index = 0; index < png.data.length; index += 4) {
87+
png.data[index] = 255;
88+
png.data[index + 1] = 255;
89+
png.data[index + 2] = 255;
90+
png.data[index + 3] = 255;
91+
}
92+
fs.writeFileSync(filePath, PNG.sync.write(png));
93+
}
94+
95+
function makeAndroidTimeoutEvidenceSession(sessionName: string): SessionStore {
96+
const sessionStore = makeSessionStore();
97+
const session = makeSession(sessionName, androidDevice);
98+
session.snapshot = {
99+
nodes: [
100+
{
101+
ref: 'e1',
102+
index: 0,
103+
depth: 0,
104+
type: 'android.widget.Button',
105+
label: 'Continue',
106+
hittable: true,
107+
rect: { x: 20, y: 40, width: 120, height: 48 },
108+
},
109+
],
110+
createdAt: Date.now(),
111+
backend: 'android',
112+
};
113+
sessionStore.set(sessionName, session);
114+
return sessionStore;
115+
}
116+
117+
function mockAndroidTimeoutEvidenceDispatch(): void {
118+
mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => {
119+
if (command === 'snapshot') throw androidSnapshotTimeoutError();
120+
if (command === 'screenshot') {
121+
const screenshotPath = positionals[0];
122+
expect(context?.screenshotNoStabilize).toBe(true);
123+
writeSolidPng(screenshotPath);
124+
return { path: screenshotPath };
125+
}
126+
return {};
127+
});
128+
}
129+
130+
function androidSnapshotTimeoutError(): AppError {
131+
return new AppError(
132+
'COMMAND_FAILED',
133+
'Android UI hierarchy dump timed out while waiting for the UI to become idle.',
134+
{
135+
cmd: 'adb',
136+
args: ['exec-out', 'uiautomator', 'dump', '/dev/tty'],
137+
timeoutMs: 8000,
138+
hint: 'Android accessibility snapshots can be blocked by busy or continuously changing app UI. Use screenshot as visual truth after this timeout.',
139+
},
140+
);
141+
}
142+
143+
function expectAndroidTimeoutEvidence(
144+
response: Awaited<ReturnType<typeof handleSnapshotCommands>>,
145+
) {
146+
if (!response) throw new Error('Expected snapshot response');
147+
if (response.ok) throw new Error('Expected snapshot timeout failure');
148+
expect(response.error.message).toMatch(/UI hierarchy dump timed out/i);
149+
expect(response.error.hint).toMatch(/Use screenshot as visual truth/i);
150+
assertAndroidTimeoutEvidencePayload(response.error.details?.androidSnapshotTimeoutScreenshot);
151+
}
152+
153+
function assertAndroidTimeoutEvidencePayload(evidence: unknown) {
154+
if (!evidence || typeof evidence !== 'object') {
155+
throw new Error('Expected Android snapshot timeout screenshot evidence');
156+
}
157+
const record = evidence as Record<string, unknown>;
158+
expect(record.path).toEqual(expect.stringContaining('snapshot-timeout-overlay-refs.png'));
159+
expect(fs.existsSync(record.path as string)).toBe(true);
160+
expect(record.overlayRefsAnnotated).toBe(true);
161+
expect(record.overlayRefCount).toBe(1);
162+
expect(record.overlayRefs).toEqual([expect.objectContaining({ ref: 'e1', label: 'Continue' })]);
163+
}
164+
83165
async function runWaitCommand(
84166
sessionName: string,
85167
device: SessionState['device'],
@@ -268,6 +350,26 @@ test('snapshot surfaces filtered-to-zero Android guidance for interactive snapsh
268350
}
269351
});
270352

353+
test('snapshot timeout captures Android screenshot evidence with overlay refs', async () => {
354+
const sessionName = 'android-timeout-evidence';
355+
const sessionStore = makeAndroidTimeoutEvidenceSession(sessionName);
356+
mockAndroidTimeoutEvidenceDispatch();
357+
const response = await handleSnapshotCommands({
358+
req: {
359+
token: 't',
360+
session: sessionName,
361+
command: 'snapshot',
362+
positionals: [],
363+
flags: {},
364+
},
365+
sessionName,
366+
logPath: '/tmp/daemon.log',
367+
sessionStore,
368+
});
369+
expectAndroidTimeoutEvidence(response);
370+
expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['snapshot', 'screenshot']);
371+
});
372+
271373
test('snapshot warns when recent snapshot node count collapses sharply', async () => {
272374
const sessionStore = makeSessionStore();
273375
const sessionName = 'android-stale-collapse';

‎src/daemon/snapshot-runtime.ts‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
} from './handlers/snapshot-session.ts';
1616
import { createDaemonRuntimePolicy } from './runtime-policy.ts';
1717
import { createDaemonRuntimeSessionStore } from './runtime-session.ts';
18+
import { maybeBuildAndroidSnapshotTimeoutFailure } from './android-snapshot-timeout-evidence.ts';
1819

1920
export async function dispatchSnapshotViaRuntime(params: {
2021
req: DaemonRequest;
@@ -125,12 +126,25 @@ async function dispatchSnapshotRuntimeCommand(
125126
device,
126127
snapshotScope: resolvedScope.scope,
127128
});
128-
const result = await params.execute({
129-
runtime,
130-
sessionName,
131-
req,
132-
snapshotScope: resolvedScope.scope,
133-
});
129+
let result: Awaited<ReturnType<SnapshotRuntimeCommandParams['execute']>>;
130+
try {
131+
result = await params.execute({
132+
runtime,
133+
sessionName,
134+
req,
135+
snapshotScope: resolvedScope.scope,
136+
});
137+
} catch (error) {
138+
const timeoutResponse = await maybeBuildAndroidSnapshotTimeoutFailure({
139+
error,
140+
command: params.command,
141+
logPath,
142+
session,
143+
device,
144+
});
145+
if (!timeoutResponse) throw error;
146+
return timeoutResponse;
147+
}
134148
recordSnapshotRuntimeAction({
135149
req,
136150
sessionName,

0 commit comments

Comments
 (0)