From d5f59aeacbe8d77c31bf0a0e3c84c646d29ec560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 30 May 2026 18:45:24 +0200 Subject: [PATCH] fix: improve React Navigation Maestro reliability --- .../SnapshotInstrumentation.java | 73 ++++- src/__tests__/cli-grammar.test.ts | 9 + src/__tests__/runtime-interactions.test.ts | 30 ++ src/client-types.ts | 11 + src/client.ts | 1 + src/command-catalog.ts | 2 +- src/commands/cli-grammar/capture.ts | 7 + src/commands/cli-grammar/gesture.ts | 22 +- src/commands/interaction-command-contracts.ts | 12 + src/commands/interaction-command-metadata.ts | 26 +- src/commands/interaction-gestures.ts | 59 ++++ .../__tests__/runtime-interactions.test.ts | 33 +- .../maestro/__tests__/runtime-targets.test.ts | 279 ++++++++++++++++ src/compat/maestro/runtime-assertions.ts | 2 + src/compat/maestro/runtime-interactions.ts | 103 ++++-- src/compat/maestro/runtime-support.ts | 18 ++ src/compat/maestro/runtime-targets.ts | 135 +++++++- src/compat/maestro/support-matrix.ts | 2 +- .../__tests__/dispatch-interactions.test.ts | 44 +++ src/core/__tests__/dispatch-open.test.ts | 76 ++++- src/core/dispatch-interactions.ts | 64 +++- src/core/dispatch.ts | 38 ++- src/core/scroll-gesture.ts | 95 ++++++ src/core/settings-contract.ts | 4 +- .../__tests__/request-handler-catalog.test.ts | 4 +- .../__tests__/session-replay-vars.test.ts | 13 +- src/daemon/handlers/__tests__/session.test.ts | 13 + .../handlers/session-replay-action-runtime.ts | 13 + src/daemon/handlers/session-replay-runtime.ts | 8 +- src/daemon/handlers/session-replay.ts | 2 + src/daemon/handlers/session-test-runtime.ts | 90 ++++++ src/daemon/handlers/session-test-types.ts | 1 + src/daemon/request-generic-dispatch.ts | 4 +- src/daemon/touch-reference-frame.ts | 43 +-- src/platforms/android/__tests__/index.test.ts | 305 +++++++++++++++++- .../__tests__/snapshot-helper-session.test.ts | 48 ++- .../android/__tests__/snapshot.test.ts | 38 ++- src/platforms/android/app-lifecycle.ts | 66 ++++ src/platforms/android/device-input-state.ts | 6 +- src/platforms/android/settings.ts | 39 +++ .../android/snapshot-helper-session.ts | 88 +++-- .../android/snapshot-helper-types.ts | 4 +- src/platforms/android/snapshot.ts | 46 ++- src/platforms/android/ui-hierarchy.ts | 97 +++++- src/platforms/ios/__tests__/index.test.ts | 50 +++ src/platforms/ios/apps.ts | 39 ++- src/replay/__tests__/script.test.ts | 10 +- src/utils/__tests__/args.test.ts | 3 +- .../__tests__/selector-is-predicates.test.ts | 23 ++ src/utils/cli-command-overrides.ts | 10 +- src/utils/cli-help.ts | 2 +- src/utils/selector-is-predicates.ts | 3 + src/utils/snapshot.ts | 1 + .../android-lifecycle.test.ts | 12 +- .../provider-scenarios/android-world.ts | 127 +++++++- website/docs/docs/commands.md | 3 + website/docs/docs/quick-start.md | 1 + website/docs/docs/replay-e2e.md | 5 +- 58 files changed, 2147 insertions(+), 215 deletions(-) diff --git a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 9e5020c78..6b4bac4bd 100644 --- a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -29,10 +29,10 @@ public final class SnapshotInstrumentation extends Instrumentation { private static final String OUTPUT_FORMAT = "uiautomator-xml"; private static final String HELPER_API_VERSION = "1"; private static final int CHUNK_SIZE = 2 * 1024; - // Keep the default quiet window short: RN/animation-heavy apps often never become fully idle, - // and callers can still override this for alert-style flows that need a longer settle period. - private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 25; - private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 25; + // Match the host defaults: long enough to avoid mid-transition RN snapshots, but still bounded + // below the stock uiautomator idle wait so busy apps do not stall every capture. + private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 500; + private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 100; private static final long DEFAULT_TIMEOUT_MS = 8_000; private static final int DEFAULT_MAX_DEPTH = 128; private static final int DEFAULT_MAX_NODES = 5_000; @@ -351,7 +351,7 @@ private CaptureResult captureXml( AccessibilityNodeInfo root = automation.getRootInActiveWindow(); try { if (root != null) { - appendNode(xml, root, 0, 0, maxDepth, maxNodes, stats); + appendNode(xml, root, 0, 0, maxDepth, maxNodes, stats, null); windowCount = 1; } captureMode = "active-window"; @@ -439,7 +439,15 @@ private static int appendInteractiveWindowRoots( } StringBuilder windowXml = new StringBuilder(); CaptureStats windowStats = stats.copy(); - appendNode(windowXml, root, windowCount, 0, maxDepth, maxNodes, windowStats); + appendNode( + windowXml, + root, + windowCount, + 0, + maxDepth, + maxNodes, + windowStats, + readWindowMetadata(window, windowCount)); xml.append(windowXml); stats.copyFrom(windowStats); windowCount += 1; @@ -482,7 +490,8 @@ private static void appendNode( int depth, int maxDepth, int maxNodes, - CaptureStats stats) { + CaptureStats stats, + WindowMetadata windowMetadata) { if (stats.nodeCount >= maxNodes) { stats.truncated = true; return; @@ -495,11 +504,15 @@ private static void appendNode( // without affecting current snapshot semantics; add fields back here when TS starts reading // them. appendAttribute(xml, "index", Integer.toString(nodeIndex)); + if (windowMetadata != null) { + appendWindowMetadata(xml, windowMetadata); + } appendNonEmptyAttribute(xml, "text", node.getText()); appendNonEmptyAttribute(xml, "resource-id", node.getViewIdResourceName()); appendAttribute(xml, "class", node.getClassName()); appendNonEmptyAttribute(xml, "package", node.getPackageName()); appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription()); + appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); appendTrueAttribute(xml, "clickable", node.isClickable()); appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled())); appendTrueAttribute(xml, "focusable", node.isFocusable()); @@ -550,7 +563,7 @@ private static void appendNode( continue; } try { - appendNode(xml, child, index, depth + 1, maxDepth, maxNodes, stats); + appendNode(xml, child, index, depth + 1, maxDepth, maxNodes, stats, null); } finally { child.recycle(); } @@ -571,6 +584,32 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean } } + private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metadata) { + appendAttribute(xml, "window-index", Integer.toString(metadata.index)); + appendAttribute(xml, "window-type", Integer.toString(metadata.type)); + appendAttribute(xml, "window-layer", Integer.toString(metadata.layer)); + appendAttribute(xml, "window-active", Boolean.toString(metadata.active)); + appendAttribute(xml, "window-focused", Boolean.toString(metadata.focused)); + appendAttribute( + xml, + "window-bounds", + String.format( + Locale.ROOT, + "[%d,%d][%d,%d]", + metadata.bounds.left, + metadata.bounds.top, + metadata.bounds.right, + metadata.bounds.bottom)); + } + + @SuppressWarnings("deprecation") + private static WindowMetadata readWindowMetadata(AccessibilityWindowInfo window, int index) { + Rect bounds = new Rect(); + window.getBoundsInScreen(bounds); + return new WindowMetadata( + index, window.getType(), window.getLayer(), window.isActive(), window.isFocused(), bounds); + } + private static void appendAttribute(StringBuilder xml, String name, CharSequence value) { String stringValue = value == null ? "" : value.toString(); xml.append(' '); @@ -702,4 +741,22 @@ private static final class CaptureResult { this.truncated = truncated; } } + + private static final class WindowMetadata { + final int index; + final int type; + final int layer; + final boolean active; + final boolean focused; + final Rect bounds; + + WindowMetadata(int index, int type, int layer, boolean active, boolean focused, Rect bounds) { + this.index = index; + this.type = type; + this.layer = layer; + this.active = active; + this.focused = focused; + this.bounds = bounds; + } + } } diff --git a/src/__tests__/cli-grammar.test.ts b/src/__tests__/cli-grammar.test.ts index c1de4e3d2..d05e8dd1d 100644 --- a/src/__tests__/cli-grammar.test.ts +++ b/src/__tests__/cli-grammar.test.ts @@ -79,4 +79,13 @@ test('settings grammar owns positional parsing for CLI commands', () => { assert.equal(location.state, 'set'); assert.equal(location.latitude, 37.3349); assert.equal(location.longitude, -122.009); + + const clearAppState = readInputFromCli('settings', ['clear-app-state', 'com.example.app'], { + ...BASE_FLAGS, + platform: 'android', + }); + assert.equal(clearAppState.platform, 'android'); + assert.equal(clearAppState.setting, 'clear-app-state'); + assert.equal(clearAppState.state, 'clear'); + assert.equal(clearAppState.app, 'com.example.app'); }); diff --git a/src/__tests__/runtime-interactions.test.ts b/src/__tests__/runtime-interactions.test.ts index 2a44ed919..6d0f8a11b 100644 --- a/src/__tests__/runtime-interactions.test.ts +++ b/src/__tests__/runtime-interactions.test.ts @@ -589,6 +589,36 @@ test('runtime directional swipe uses the visible viewport instead of off-screen assert.deepEqual(calls, [{ from: { x: 50, y: 50 }, to: { x: 25, y: 50 } }]); }); +test('runtime gesture swipe presets use stable viewport lanes', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(snapshotWithOffscreenContent(), { + platform: 'android', + swipe: async (_context, from, to, options) => { + calls.push({ from, to, durationMs: options?.durationMs }); + }, + }); + + const pageSwipe = await device.interactions.swipe({ + preset: 'left', + durationMs: 300, + session: 'default', + }); + const edgeSwipe = await device.interactions.swipe({ + preset: 'right-edge', + durationMs: 350, + session: 'default', + }); + + assert.deepEqual(pageSwipe.from, { x: 90, y: 65 }); + assert.deepEqual(pageSwipe.to, { x: 10, y: 65 }); + assert.deepEqual(edgeSwipe.from, { x: 8, y: 50 }); + assert.deepEqual(edgeSwipe.to, { x: 85, y: 50 }); + assert.deepEqual(calls, [ + { from: { x: 90, y: 65 }, to: { x: 10, y: 65 }, durationMs: 300 }, + { from: { x: 8, y: 50 }, to: { x: 85, y: 50 }, durationMs: 350 }, + ]); +}); + test('runtime viewport gestures reject inspect-only macOS surfaces', async () => { for (const surface of ['desktop', 'menubar'] as const) { const device = createInteractionDevice(selectorSnapshot(), { diff --git a/src/client-types.ts b/src/client-types.ts index 6a61f9a67..d22c4ed5e 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -595,6 +595,11 @@ export type FlingOptions = ClientCommandBaseOptions & { durationMs?: number; }; +export type SwipeGestureOptions = ClientCommandBaseOptions & { + preset: 'left' | 'right' | 'left-edge' | 'right-edge'; + durationMs?: number; +}; + export type FocusOptions = ClientCommandBaseOptions & { x: number; y: number; @@ -763,6 +768,11 @@ export type PermissionTarget = | 'input-monitoring'; export type SettingsUpdateOptions = + | (ClientCommandBaseOptions & { + setting: 'clear-app-state'; + state: 'clear'; + app?: string; + }) | (ClientCommandBaseOptions & { setting: 'wifi' | 'airplane' | 'location'; state: 'on' | 'off'; @@ -915,6 +925,7 @@ export type AgentDeviceClient = { swipe: (options: SwipeOptions) => Promise; pan: (options: PanOptions) => Promise; fling: (options: FlingOptions) => Promise; + swipeGesture: (options: SwipeGestureOptions) => Promise; focus: (options: FocusOptions) => Promise; type: (options: TypeTextOptions) => Promise; fill: (options: FillOptions) => Promise; diff --git a/src/client.ts b/src/client.ts index 48d1068b0..1bf2abe79 100644 --- a/src/client.ts +++ b/src/client.ts @@ -270,6 +270,7 @@ export function createAgentDeviceClient( swipe: async (options) => await executeCommand('swipe', options), pan: async (options) => await executeCommand('gesture-pan', options), fling: async (options) => await executeCommand('gesture-fling', options), + swipeGesture: async (options) => await executeCommand('gesture-swipe', options), focus: async (options) => await executeCommand('focus', options), type: async (options) => await executeCommand('type', options), fill: async (options) => await executeCommand('fill', options), diff --git a/src/command-catalog.ts b/src/command-catalog.ts index 33d1e4051..3d2dd42f8 100644 --- a/src/command-catalog.ts +++ b/src/command-catalog.ts @@ -66,7 +66,7 @@ const LOCAL_CLI_COMMANDS = { session: 'session', } as const; -const GESTURE_SUBCOMMANDS = ['pan', 'fling', 'pinch', 'rotate', 'transform'] as const; +const GESTURE_SUBCOMMANDS = ['pan', 'fling', 'swipe', 'pinch', 'rotate', 'transform'] as const; export const GESTURE_SUBCOMMAND_ERROR = `gesture requires one of: ${GESTURE_SUBCOMMANDS.join(', ')}`; export type PublicCommandName = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; diff --git a/src/commands/cli-grammar/capture.ts b/src/commands/cli-grammar/capture.ts index b0ca03b7a..2da3e1e54 100644 --- a/src/commands/cli-grammar/capture.ts +++ b/src/commands/cli-grammar/capture.ts @@ -231,10 +231,17 @@ function readSettingsOptionsFromPositionals( mode: readPermissionMode(positionals[3]), }; } + if (setting === 'clear-app-state') { + const app = state === 'clear' ? positionals[2] : state; + return { ...base, setting, state: 'clear', app }; + } throw new AppError('INVALID_ARGS', 'Invalid settings arguments.'); } function settingsPositionals(input: SettingsUpdateOptions): string[] { + if (input.setting === 'clear-app-state') { + return [input.setting, ...optionalString(input.app)]; + } if (input.setting === 'location' && input.state === 'set') { return [input.setting, input.state, String(input.latitude), String(input.longitude)]; } diff --git a/src/commands/cli-grammar/gesture.ts b/src/commands/cli-grammar/gesture.ts index 6fe2c7b75..b8f67f8c0 100644 --- a/src/commands/cli-grammar/gesture.ts +++ b/src/commands/cli-grammar/gesture.ts @@ -21,6 +21,7 @@ export const gestureDaemonWriters = { 'gesture-fling': direct(PUBLIC_COMMANDS.gesture, (input) => flingPositionals(input as FlingOptions), ), + 'gesture-swipe': direct(PUBLIC_COMMANDS.gesture, swipePresetPositionals), 'gesture-pinch': direct(PUBLIC_COMMANDS.gesture, pinchPositionals), 'gesture-rotate': direct(PUBLIC_COMMANDS.gesture, (input) => rotateGesturePositionals(input as RotateGestureOptions), @@ -49,6 +50,8 @@ function gesturePositionals(input: CommandInput): string[] { ...optionalNumber(input.distance), ...optionalNumber(input.durationMs), ]; + case 'swipe': + return swipePresetPositionals(input); case 'pinch': return [ 'pinch', @@ -78,11 +81,19 @@ function gesturePositionals(input: CommandInput): string[] { default: throw new AppError( 'INVALID_ARGS', - 'gesture requires pan, fling, pinch, rotate, or transform', + 'gesture requires pan, fling, swipe, pinch, rotate, or transform', ); } } +function swipePresetPositionals(input: CommandInput): string[] { + return [ + 'swipe', + requiredDaemonString(input.preset, 'gesture swipe requires preset'), + ...optionalNumber(input.durationMs), + ]; +} + function panPositionals(input: CommandInput): string[] { return [ 'pan', @@ -153,6 +164,13 @@ function gestureInputFromCli(positionals: string[], flags: CliFlags): Record, +): Promise { + if (options.from || options.to || options.direction || options.distance !== undefined) { + throw new AppError( + 'INVALID_ARGS', + 'gesture swipe preset cannot be combined with from, to, direction, or distance', + ); + } + const preset = parseSwipePreset(options.preset); + await assertSupportedInteractionSurface(runtime, options, 'swipe'); + const capture = await captureInteractionSnapshot(runtime, options, false); + const frame = resolveSnapshotReferenceFrame(capture.snapshot.nodes); + const plan = buildSwipePresetGesturePlan(preset, frame, { platform: runtime.backend.platform }); + const durationMs = + options.durationMs === undefined + ? undefined + : requireIntInRange(options.durationMs, 'durationMs', 16, 10_000); + const from = { x: plan.x1, y: plan.y1 }; + const to = { x: plan.x2, y: plan.y2 }; + const backendResult = await swipeBackend(toBackendContext(runtime, options), from, to, { + durationMs, + }); + const formattedBackendResult = toBackendResult(backendResult); + return { + kind: 'swipe', + from, + to, + preset, + ...(durationMs !== undefined ? { durationMs } : {}), + fromTarget: { kind: 'viewport' }, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText(`Swiped ${preset}`), + }; +} + export const pinchCommand: RuntimeCommand = async ( runtime, options, @@ -491,6 +539,17 @@ function resolveSnapshotViewport(nodes: SnapshotState['nodes']): Rect { }; } +function resolveSnapshotReferenceFrame(nodes: SnapshotState['nodes']): { + referenceWidth: number; + referenceHeight: number; +} { + const viewport = resolveSnapshotViewport(nodes); + return { + referenceWidth: viewport.width, + referenceHeight: viewport.height, + }; +} + function isUsableRect(rect: SnapshotNode['rect']): rect is NonNullable { return Boolean(rect && rect.width > 0 && rect.height > 0); } diff --git a/src/compat/maestro/__tests__/runtime-interactions.test.ts b/src/compat/maestro/__tests__/runtime-interactions.test.ts index e18754fb3..bbde33420 100644 --- a/src/compat/maestro/__tests__/runtime-interactions.test.ts +++ b/src/compat/maestro/__tests__/runtime-interactions.test.ts @@ -38,8 +38,8 @@ test('invokeMaestroTapOn resolves mutating taps from the current raw snapshot', expect(clicks).toEqual([['86', '89']]); }); -test('invokeMaestroSwipeScreen uses an Android content-lane directional swipe', async () => { - const swipes: string[][] = []; +test('invokeMaestroSwipeScreen maps horizontal directional swipes to native gesture presets', async () => { + const gestures: string[][] = []; const response = await invokeMaestroSwipeScreen({ baseReq: { token: 'test', @@ -48,11 +48,30 @@ test('invokeMaestroSwipeScreen uses an Android content-lane directional swipe', }, positionals: ['direction', 'left', '300'], invoke: async (req: DaemonRequest): Promise => { - if (req.command === 'snapshot') { - return { ok: true, data: fullScreenSnapshot(1080, 2340) }; + if (req.command === 'gesture') { + gestures.push(req.positionals ?? []); + return { ok: true, data: {} }; } - if (req.command === 'swipe') { - swipes.push(req.positionals ?? []); + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + expect(response.ok).toBe(true); + expect(gestures).toEqual([['swipe', 'left', '300']]); +}); + +test('invokeMaestroSwipeScreen mirrors horizontal directional swipe presets', async () => { + const gestures: string[][] = []; + const response = await invokeMaestroSwipeScreen({ + baseReq: { + token: 'test', + session: 'pager', + flags: { platform: 'android' }, + }, + positionals: ['direction', 'right', '300'], + invoke: async (req: DaemonRequest): Promise => { + if (req.command === 'gesture') { + gestures.push(req.positionals ?? []); return { ok: true, data: {} }; } return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; @@ -60,7 +79,7 @@ test('invokeMaestroSwipeScreen uses an Android content-lane directional swipe', }); expect(response.ok).toBe(true); - expect(swipes).toEqual([['864', '1521', '216', '1521', '300']]); + expect(gestures).toEqual([['swipe', 'right', '300']]); }); test('invokeMaestroSwipeScreen preserves vertical percentage endpoints', async () => { diff --git a/src/compat/maestro/__tests__/runtime-targets.test.ts b/src/compat/maestro/__tests__/runtime-targets.test.ts index 49b5f8e68..5bdf8c886 100644 --- a/src/compat/maestro/__tests__/runtime-targets.test.ts +++ b/src/compat/maestro/__tests__/runtime-targets.test.ts @@ -59,6 +59,64 @@ test('resolveMaestroNodeFromSnapshot blocks taps on app content behind React Nat }); }); +test('resolveMaestroNodeFromSnapshot does not match plain text as a substring', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.Button', + label: 'Push feed', + rect: { x: 32, y: 320, width: 280, height: 96 }, + enabled: true, + hittable: true, + depth: 5, + }, + { + index: 2, + ref: 'e2', + type: 'StaticText', + label: 'Albums, back', + rect: { x: 120, y: 80, width: 180, height: 48 }, + depth: 5, + }, + ], + }; + + const plain = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Feed" || text="Feed" || id="Feed"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + const regex = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label=".*feed" || text=".*feed" || id=".*feed"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + const composite = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Albums" || text="Albums" || id="Albums"', + 'ios', + { referenceWidth: 402, referenceHeight: 874 }, + ); + + expect(plain).toMatchObject({ + ok: false, + message: expect.stringContaining('Feed'), + }); + expect(regex).toMatchObject({ + ok: true, + node: expect.objectContaining({ label: 'Push feed' }), + }); + expect(composite).toMatchObject({ + ok: true, + node: expect.objectContaining({ label: 'Albums, back' }), + }); +}); + test('resolveVisibleMaestroNodeFromSnapshot does not block content behind collapsed React Native warnings', () => { const snapshot: SnapshotState = { createdAt: Date.now(), @@ -277,6 +335,184 @@ test('resolveMaestroNodeFromSnapshot prefers duplicate text on foreground overla }); }); +test('resolveMaestroNodeFromSnapshot prefers full-width screen over stale side navigation duplicate', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.ScrollView', + label: 'Article, Go back, Replace params', + rect: { x: 0, y: 319, width: 1080, height: 2021 }, + depth: 17, + }, + { + index: 2, + ref: 'e2', + type: 'android.widget.Button', + label: 'Go back', + rect: { x: 33, y: 667, width: 1014, height: 110 }, + enabled: true, + hittable: true, + depth: 19, + parentIndex: 1, + }, + { + index: 30, + ref: 'e30', + type: 'android.widget.ScrollView', + label: 'Albums, Go back, Go to Article', + rect: { x: 0, y: 319, width: 816, height: 2021 }, + depth: 17, + }, + { + index: 31, + ref: 'e31', + type: 'android.widget.Button', + label: 'Go back', + rect: { x: 0, y: 581, width: 783, height: 110 }, + enabled: true, + hittable: true, + depth: 19, + parentIndex: 30, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Go back" || text="Go back" || id="Go back"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + { promoteTapTarget: true }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 2 }), + rect: { x: 33, y: 667, width: 1014, height: 110 }, + }); +}); + +test('resolveMaestroNodeFromSnapshot uses visible assertion context for equal overlapping screens', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 6, + ref: 'e6', + type: 'android.view.View', + label: 'Albums', + value: 'Albums', + rect: { x: 154, y: 194, width: 188, height: 74 }, + depth: 22, + parentIndex: 25, + }, + { + index: 7, + ref: 'e7', + type: 'android.widget.ScrollView', + rect: { x: 0, y: 319, width: 1080, height: 2021 }, + depth: 17, + }, + { + index: 8, + ref: 'e8', + type: 'android.widget.Button', + label: 'Push albums', + rect: { x: 33, y: 352, width: 361, height: 110 }, + enabled: true, + hittable: true, + depth: 19, + parentIndex: 7, + }, + { + index: 12, + ref: 'e12', + type: 'android.widget.Button', + label: 'Push article', + rect: { x: 33, y: 495, width: 340, height: 110 }, + enabled: true, + hittable: true, + depth: 19, + parentIndex: 7, + }, + { + index: 13, + ref: 'e13', + type: 'android.widget.TextView', + label: 'Push article', + value: 'Push article', + rect: { x: 99, y: 523, width: 208, height: 55 }, + depth: 20, + parentIndex: 12, + }, + { + index: 25, + ref: 'e25', + type: 'android.widget.ScrollView', + rect: { x: 0, y: 319, width: 1080, height: 2021 }, + depth: 17, + }, + { + index: 26, + ref: 'e26', + type: 'android.widget.Button', + label: 'Push article', + rect: { x: 33, y: 352, width: 340, height: 110 }, + enabled: true, + hittable: true, + depth: 19, + parentIndex: 25, + }, + { + index: 27, + ref: 'e27', + type: 'android.widget.TextView', + label: 'Push article', + value: 'Push article', + rect: { x: 99, y: 380, width: 208, height: 55 }, + depth: 20, + parentIndex: 26, + }, + { + index: 30, + ref: 'e30', + type: 'android.widget.Button', + label: 'Push albums', + rect: { x: 33, y: 495, width: 361, height: 110 }, + enabled: true, + hittable: true, + depth: 19, + parentIndex: 25, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Push article" || text="Push article" || id="Push article"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + { + promoteTapTarget: true, + preferredContext: { + node: snapshot.nodes[0]!, + rect: snapshot.nodes[0]!.rect!, + }, + }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 12 }), + rect: { x: 33, y: 495, width: 340, height: 110 }, + }); +}); + test('resolveVisibleMaestroNodeFromSnapshot requires visible text matches to be on screen', () => { const snapshot: SnapshotState = { createdAt: Date.now(), @@ -423,6 +659,49 @@ test('resolveMaestroNodeFromSnapshot prefers concrete Android tab rect over hidd }); }); +test('resolveMaestroNodeFromSnapshot ignores Android nodes hidden from users', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.Button', + label: 'Settings', + rect: { x: 0, y: 0, width: 200, height: 80 }, + enabled: true, + hittable: true, + visibleToUser: false, + }, + { + index: 2, + ref: 'e2', + type: 'android.widget.Button', + label: 'Settings', + rect: { x: 300, y: 700, width: 200, height: 80 }, + enabled: true, + hittable: true, + visibleToUser: true, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Settings"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + { promoteTapTarget: true }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 2 }), + rect: { x: 300, y: 700, width: 200, height: 80 }, + }); +}); + test('resolveMaestroNodeFromSnapshot prefers exact Android tab label over normalized header icon text', () => { const snapshot: SnapshotState = { createdAt: Date.now(), diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts index 853608488..ee8a0d7fb 100644 --- a/src/compat/maestro/runtime-assertions.ts +++ b/src/compat/maestro/runtime-assertions.ts @@ -6,6 +6,7 @@ import { sleep } from '../../utils/timeouts.ts'; import { captureMaestroRawSnapshot, errorResponse, + rememberMaestroVisibleContext, readSnapshotState, type MaestroRuntimeInvoke, type ReplayBaseRequest, @@ -134,6 +135,7 @@ async function readMaestroVisibilitySample( infrastructureFailure: false, }; } + rememberMaestroVisibleContext(params.scope, selector); return { visible: true, response: { diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index fbe0ee5a6..cf1e8ed46 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -7,12 +7,15 @@ import { type ScrollDirection, } from '../../core/scroll-gesture.ts'; import type { ReplayVarScope } from '../../replay/vars.ts'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from '../../utils/timeouts.ts'; import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; import { captureMaestroRawSnapshot, + clearMaestroVisibleContext, errorResponse, readCachedMaestroReferenceFrame, + readMaestroVisibleContext, readSnapshotState, type FailedDaemonResponse, type MaestroRuntimeInvoke, @@ -23,6 +26,8 @@ import { readMaestroSelectorPlatform, resolveMaestroFuzzyTextNodeFromSnapshot, resolveMaestroNodeFromSnapshot, + resolveVisibleMaestroNodeFromSnapshot, + type MaestroPreferredContext, type MaestroSnapshotTarget, type MaestroTapOnOptions, } from './runtime-targets.ts'; @@ -142,12 +147,28 @@ export async function invokeMaestroSwipeScreen(params: { invoke: MaestroRuntimeInvoke; scope?: ReplayVarScope; }): Promise { + const presetResponse = await maybeInvokeMaestroDirectionalSwipePreset(params); + if (presetResponse) return presetResponse; const swipe = await resolveMaestroScreenSwipe(params); if (!swipe.ok) return swipe.response; return await invokeSwipeGesture(params, swipe, swipe.durationMs); } +async function maybeInvokeMaestroDirectionalSwipePreset(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [mode, direction, durationMs] = params.positionals; + if (mode !== 'direction' || (direction !== 'left' && direction !== 'right')) return undefined; + return await params.invoke({ + ...params.baseReq, + command: 'gesture', + positionals: ['swipe', direction, ...(durationMs ? [durationMs] : [])], + }); +} + export async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { const [selector, rawOptions] = params.positionals; if (!selector) { @@ -225,11 +246,7 @@ async function resolveMaestroScreenSwipe(params: { const [mode, ...args] = params.positionals; if (mode === 'direction') { - return resolveDirectionalScreenSwipe( - args, - frame, - readMaestroSelectorPlatform(params.baseReq.flags), - ); + return resolveDirectionalScreenSwipe(args, frame); } if (mode === 'percent') { return resolvePercentScreenSwipe( @@ -258,7 +275,6 @@ async function captureFrameForMaestroScreenSwipe(params: { function resolveDirectionalScreenSwipe( args: string[], frame: { referenceWidth: number; referenceHeight: number }, - platform: string, ): MaestroScreenSwipeResolution { const [direction, durationMs] = args; if (!direction) { @@ -270,9 +286,7 @@ function resolveDirectionalScreenSwipe( switch (direction) { case 'up': case 'down': - case 'left': - case 'right': - return buildMaestroDirectionalScreenSwipe(direction, frame, platform, durationMs); + return buildMaestroDirectionalScreenSwipe(direction, frame, durationMs); default: return { ok: false, @@ -287,7 +301,6 @@ function resolveDirectionalScreenSwipe( function buildMaestroDirectionalScreenSwipe( direction: ScrollDirection, frame: { referenceWidth: number; referenceHeight: number }, - platform: string, durationMs: string | undefined, ): MaestroScreenSwipeResolution { const plan = buildSwipeGesturePlan({ @@ -298,13 +311,6 @@ function buildMaestroDirectionalScreenSwipe( }); const start = clampGesturePoint({ x: plan.x1, y: plan.y1 }, frame, 8); const end = clampGesturePoint({ x: plan.x2, y: plan.y2 }, frame, 8); - - if ((direction === 'left' || direction === 'right') && platform === 'android') { - const contentLaneY = pointFromPercent(frame, 50, 65, { marginPx: 8 }).y; - start.y = contentLaneY; - end.y = contentLaneY; - } - return { ok: true, start, @@ -429,11 +435,29 @@ async function invokeMaestroSnapshotTapOn( target.target, extractMaestroVisibleTextQuery(selector) !== null, ); + emitDiagnostic({ + level: 'debug', + phase: 'maestro_tap_target', + data: { + selector, + node: { + index: target.target.node.index, + type: target.target.node.type, + label: target.target.node.label, + value: target.target.node.value, + identifier: target.target.node.identifier, + visibleToUser: target.target.node.visibleToUser, + }, + rect: target.target.rect, + point, + }, + }); const response = await params.invoke({ ...params.baseReq, command: 'click', positionals: [String(point.x), String(point.y)], }); + if (response.ok) clearMaestroVisibleContext(params.scope); return { response, targetResolved: true, @@ -485,22 +509,21 @@ async function resolveMaestroSnapshotTarget( } const frame = getSnapshotReferenceFrame(snapshot); - const resolution = resolveMaestroNodeFromSnapshot( - snapshot, - selector, - options, - readMaestroSelectorPlatform(params.baseReq.flags), - frame, - resolutionOptions, - ); + const platform = readMaestroSelectorPlatform(params.baseReq.flags); + const preferredContext = resolvePreferredMaestroContext(params, snapshot, platform, frame); + const resolution = resolveMaestroNodeFromSnapshot(snapshot, selector, options, platform, frame, { + ...resolutionOptions, + preferredContext, + }); if (!resolution.ok) { const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); if (fuzzyTextQuery) { const fuzzyResolution = resolveMaestroFuzzyTextNodeFromSnapshot( snapshot, fuzzyTextQuery, + platform, frame, - resolutionOptions, + { ...resolutionOptions, preferredContext }, ); if (fuzzyResolution.ok) { return { @@ -534,6 +557,34 @@ async function resolveMaestroSnapshotTarget( }; } +function resolvePreferredMaestroContext( + params: { baseReq: ReplayBaseRequest; scope?: ReplayVarScope }, + snapshot: NonNullable>, + platform: ReturnType, + frame: ReturnType, +): MaestroPreferredContext | undefined { + const context = readMaestroVisibleContext(params.scope); + if (!context) return undefined; + const target = resolveVisibleMaestroNodeFromSnapshot(snapshot, context.selector, platform, frame); + if (!target.ok) return undefined; + emitDiagnostic({ + level: 'debug', + phase: 'maestro_preferred_context', + data: { + selector: context.selector, + node: { + index: target.node.index, + type: target.node.type, + label: target.node.label, + value: target.node.value, + identifier: target.node.identifier, + }, + rect: target.rect, + }, + }); + return { node: target.node, rect: target.rect }; +} + function readMaestroTapOnOptions( rawOptions: string | undefined, ): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { diff --git a/src/compat/maestro/runtime-support.ts b/src/compat/maestro/runtime-support.ts index 36b1cbafb..f58f3f1ab 100644 --- a/src/compat/maestro/runtime-support.ts +++ b/src/compat/maestro/runtime-support.ts @@ -19,6 +19,7 @@ export type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; const maestroReferenceFrameCache = new WeakMap(); +const maestroVisibleContextCache = new WeakMap(); export function errorResponse( code: string, @@ -68,6 +69,23 @@ export function readCachedMaestroReferenceFrame( return scope ? maestroReferenceFrameCache.get(scope) : undefined; } +export function rememberMaestroVisibleContext( + scope: ReplayVarScope | undefined, + selector: string, +): void { + if (scope) maestroVisibleContextCache.set(scope, { selector }); +} + +export function readMaestroVisibleContext( + scope: ReplayVarScope | undefined, +): { selector: string } | undefined { + return scope ? maestroVisibleContextCache.get(scope) : undefined; +} + +export function clearMaestroVisibleContext(scope: ReplayVarScope | undefined): void { + if (scope) maestroVisibleContextCache.delete(scope); +} + function rememberMaestroReferenceFrame(scope: ReplayVarScope, data: unknown): void { const snapshot = readSnapshotState(data); const frame = getSnapshotReferenceFrame(snapshot); diff --git a/src/compat/maestro/runtime-targets.ts b/src/compat/maestro/runtime-targets.ts index 3078ccd5e..d5b63d992 100644 --- a/src/compat/maestro/runtime-targets.ts +++ b/src/compat/maestro/runtime-targets.ts @@ -45,8 +45,14 @@ type MaestroResolvedSnapshotMatch = { inheritedRect: boolean; }; +export type MaestroPreferredContext = { + node: SnapshotNode; + rect: Rect; +}; + type MaestroMatchResolutionOptions = { promoteTapTarget?: boolean; + preferredContext?: MaestroPreferredContext; }; type ReactNativeOverlayFilterResult = { @@ -82,24 +88,31 @@ export function resolveMaestroNodeFromSnapshot( ), ); } - const filteredMatches = filterReactNativeOverlayBlockedMatches(snapshot.nodes, matches); + const visibleMatchesResult = filterVisibleMaestroMatches({ + nodes: snapshot.nodes, + matches, + platform, + }); const target = selectMaestroSnapshotMatch( snapshot.nodes, - filteredMatches.matches, + visibleMatchesResult.matches, options.index, extractMaestroVisibleTextQuery(selector), frame, false, resolutionOptions.promoteTapTarget, + resolutionOptions.preferredContext, ); if (!target) { const index = options.index ?? 0; return { ok: false, - message: filteredMatches.blockedByReactNativeOverlay + message: visibleMatchesResult.blockedByReactNativeOverlay ? `Maestro selector matched ${matches.length} element(s), but React Native overlay is covering app content: ${selector}` - : `Maestro selector did not match index ${index}: ${selector}`, + : matches.length > 0 && visibleMatchesResult.matches.length === 0 + ? `Maestro selector matched ${matches.length} element(s), but none were visible: ${selector}` + : `Maestro selector did not match index ${index}: ${selector}`, }; } return { ok: true, node: target.node, rect: target.rect }; @@ -108,18 +121,25 @@ export function resolveMaestroNodeFromSnapshot( export function resolveMaestroFuzzyTextNodeFromSnapshot( snapshot: SnapshotState, query: string, + platform: Platform, frame: TouchReferenceFrame | undefined, resolutionOptions: MaestroMatchResolutionOptions = {}, ): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { const matches = findMaestroFuzzyTextMatches(snapshot, query); + const visibleMatchesResult = filterVisibleMaestroMatches({ + nodes: snapshot.nodes, + matches, + platform, + }); const target = selectMaestroSnapshotMatch( snapshot.nodes, - matches, + visibleMatchesResult.matches, undefined, query, frame, false, resolutionOptions.promoteTapTarget, + resolutionOptions.preferredContext, ); if (!target) { return { ok: false, message: `Maestro fuzzy text did not match: ${query}` }; @@ -295,7 +315,11 @@ function readMaestroTextTermValue( function textEqualsOrRegex(value: string | undefined, query: string): boolean { const text = value ?? ''; - if (normalizeText(text) === normalizeText(query)) return true; + const normalizedText = normalizeText(text); + const normalizedQuery = normalizeText(query); + if (normalizedText === normalizedQuery) return true; + if (isLeadingCompositeLabelMatch(normalizedText, normalizedQuery)) return true; + if (!looksLikeMaestroRegex(query)) return false; try { return new RegExp(query).test(text); } catch { @@ -303,6 +327,17 @@ function textEqualsOrRegex(value: string | undefined, query: string): boolean { } } +function isLeadingCompositeLabelMatch(normalizedText: string, normalizedQuery: string): boolean { + if (!normalizedText || !normalizedQuery) return false; + if (!normalizedText.startsWith(normalizedQuery)) return false; + const next = normalizedText.at(normalizedQuery.length); + return next === ',' || next === ':' || next === ';'; +} + +function looksLikeMaestroRegex(query: string): boolean { + return /(?:\.\*|\.\+|\[[^\]]+\]|\([^)]*\)|\||\^|\$|\\[dDsSwWbB])/.test(query); +} + function resolveNodeRect( nodes: SnapshotState['nodes'], node: SnapshotNode, @@ -339,6 +374,7 @@ function selectMaestroSnapshotMatch( frame: TouchReferenceFrame | undefined, requireOnScreen = false, promoteTapTarget = false, + preferredContext?: MaestroPreferredContext, ): { node: SnapshotNode; rect: Rect } | null { const nodeByIndex = buildSnapshotNodeByIndex(nodes); const candidates = resolveMaestroSnapshotMatchCandidates( @@ -356,6 +392,7 @@ function selectMaestroSnapshotMatch( index, visibleTextQuery, promoteTapTarget, + preferredContext, ); return promoteMaestroSnapshotMatch(nodes, target, nodeByIndex, promoteTapTarget, frame); } @@ -393,6 +430,7 @@ function chooseMaestroSnapshotMatch( index: number | undefined, visibleTextQuery: string | null, promoteTapTarget: boolean, + preferredContext?: MaestroPreferredContext, ): MaestroResolvedSnapshotMatch | null { if (index !== undefined) return candidates[index] ?? null; const best = selectPreferredMaestroSnapshotMatch( @@ -400,6 +438,7 @@ function chooseMaestroSnapshotMatch( candidates, visibleTextQuery, promoteTapTarget, + preferredContext, ); if (!shouldInferMaestroTabSlot(best, visibleTextQuery, promoteTapTarget)) return best; return inferMaestroMissingTabSlotMatch(nodes, best, visibleTextQuery!) ?? best; @@ -410,13 +449,14 @@ function selectPreferredMaestroSnapshotMatch( candidates: MaestroResolvedSnapshotMatch[], visibleTextQuery: string | null, promoteTapTarget: boolean, + preferredContext?: MaestroPreferredContext, ): MaestroResolvedSnapshotMatch | null { if (!promoteTapTarget || !visibleTextQuery) { - return selectBestMaestroSnapshotMatch(nodes, candidates, visibleTextQuery); + return selectBestMaestroSnapshotMatch(nodes, candidates, visibleTextQuery, preferredContext); } return ( - selectLocalizedMaestroVisibleTextMatch(nodes, candidates, visibleTextQuery) ?? - selectBestMaestroSnapshotMatch(nodes, candidates, visibleTextQuery) + selectLocalizedMaestroVisibleTextMatch(nodes, candidates, visibleTextQuery, preferredContext) ?? + selectBestMaestroSnapshotMatch(nodes, candidates, visibleTextQuery, preferredContext) ); } @@ -432,11 +472,13 @@ function selectBestMaestroSnapshotMatch( nodes: SnapshotState['nodes'], candidates: MaestroResolvedSnapshotMatch[], visibleTextQuery: string | null, + preferredContext?: MaestroPreferredContext, ): MaestroResolvedSnapshotMatch | null { const foregroundCandidates = preferForegroundContainerDuplicateMatches( nodes, candidates, visibleTextQuery, + preferredContext, ); return ( foregroundCandidates.sort((left, right) => @@ -449,6 +491,7 @@ function selectLocalizedMaestroVisibleTextMatch( nodes: SnapshotState['nodes'], candidates: MaestroResolvedSnapshotMatch[], query: string, + preferredContext?: MaestroPreferredContext, ): MaestroResolvedSnapshotMatch | null { const exactMatches = candidates.filter( (candidate) => maestroVisibleTextMatchRank(candidate.node, query) === 0, @@ -458,6 +501,7 @@ function selectLocalizedMaestroVisibleTextMatch( nodes, exactMatches, query, + preferredContext, ); if (localizedExact) return localizedExact; } @@ -467,13 +511,19 @@ function selectLocalizedMaestroVisibleTextMatch( ); if (exactMatches.length > 0 || normalizedMatches.length < 2) return null; - return selectLocalizedMaestroVisibleTextMatchFromCandidates(nodes, normalizedMatches, query); + return selectLocalizedMaestroVisibleTextMatchFromCandidates( + nodes, + normalizedMatches, + query, + preferredContext, + ); } function selectLocalizedMaestroVisibleTextMatchFromCandidates( nodes: SnapshotState['nodes'], candidates: MaestroResolvedSnapshotMatch[], query: string, + preferredContext?: MaestroPreferredContext, ): MaestroResolvedSnapshotMatch | null { const nodeByIndex = buildSnapshotNodeByIndex(nodes); const localized = candidates.filter( @@ -484,13 +534,15 @@ function selectLocalizedMaestroVisibleTextMatchFromCandidates( ), ); - return selectBestMaestroSnapshotMatch(nodes, localized, query); + return selectBestMaestroSnapshotMatch(nodes, localized, query, preferredContext); } +// fallow-ignore-next-line complexity function preferForegroundContainerDuplicateMatches( nodes: SnapshotState['nodes'], candidates: MaestroResolvedSnapshotMatch[], visibleTextQuery: string | null, + preferredContext?: MaestroPreferredContext, ): MaestroResolvedSnapshotMatch[] { if (!visibleTextQuery || candidates.length < 2) return candidates; const exact = candidates.filter( @@ -512,6 +564,17 @@ function preferForegroundContainerDuplicateMatches( ); if (overlapping.length < 2) return candidates; + const foregroundByArea = selectLargestOverlappingScreenContainerMatches(overlapping); + if (foregroundByArea.length > 0) return foregroundByArea; + + const foregroundByContext = selectContextualOverlappingScreenContainerMatches( + nodes, + overlapping, + preferredContext, + nodeByIndex, + ); + if (foregroundByContext.length > 0) return foregroundByContext; + // UIAutomator reports foreground transparent-stack screens later in the // hierarchy while preserving both screens. Prefer the later overlapping // screen only for exact duplicate text, so ordinary duplicate rows keep @@ -523,6 +586,56 @@ function preferForegroundContainerDuplicateMatches( return foreground.length > 0 ? foreground : candidates; } +function selectContextualOverlappingScreenContainerMatches( + nodes: SnapshotState['nodes'], + entries: MaestroMatchWithScreenContainer[], + context: MaestroPreferredContext | undefined, + nodeByIndex: SnapshotNodeByIndex, +): MaestroResolvedSnapshotMatch[] { + if (!context) return []; + const rawContextContainer = findMaestroScreenContainer(nodes, context.node, nodeByIndex); + const contextContainer = + rawContextContainer && rectContains(rawContextContainer.rect, context.rect) + ? rawContextContainer + : null; + const scored = entries.map((entry) => ({ + entry, + score: scoreScreenContainerAgainstContext(entry.container, context, contextContainer), + })); + const bestScore = Math.min(...scored.map((entry) => entry.score)); + if (!Number.isFinite(bestScore)) return []; + return scored.filter((entry) => entry.score === bestScore).map((entry) => entry.entry.candidate); +} + +function scoreScreenContainerAgainstContext( + container: SnapshotNode & { rect: Rect }, + context: MaestroPreferredContext, + contextContainer: (SnapshotNode & { rect: Rect }) | null, +): number { + if (contextContainer) { + if (container.index === contextContainer.index) return 0; + if (rectOverlapRatio(container.rect, contextContainer.rect) < 0.6) + return Number.POSITIVE_INFINITY; + return Math.abs(container.index - contextContainer.index); + } + + if (rectOverlapRatio(container.rect, context.rect) >= 0.6) return 0; + const orderDistance = container.index - context.node.index; + return orderDistance >= 0 ? orderDistance : 100_000 + Math.abs(orderDistance); +} + +function selectLargestOverlappingScreenContainerMatches( + entries: MaestroMatchWithScreenContainer[], +): MaestroResolvedSnapshotMatch[] { + const areas = entries.map((entry) => rectArea(entry.container.rect)); + const largestArea = Math.max(...areas); + const smallestArea = Math.min(...areas); + if (smallestArea <= 0 || largestArea < smallestArea * 1.2) return []; + return entries + .filter((entry) => rectArea(entry.container.rect) === largestArea) + .map((entry) => entry.candidate); +} + function hasOverlappingScreenContainer( entry: MaestroMatchWithScreenContainer, candidates: MaestroMatchWithScreenContainer[], diff --git a/src/compat/maestro/support-matrix.ts b/src/compat/maestro/support-matrix.ts index 73b579f46..bab225280 100644 --- a/src/compat/maestro/support-matrix.ts +++ b/src/compat/maestro/support-matrix.ts @@ -1,5 +1,5 @@ export const MAESTRO_COMPAT_SUPPORTED_CAPABILITIES = [ - 'app launch with Apple-platform launch arguments and iOS simulator clearState', + 'app launch with Apple-platform launch arguments and Android/iOS simulator clearState', 'runFlow file/inline with when.platform, when.visible, when.notVisible, and limited when.true boolean/platform expressions', 'onFlowStart and onFlowComplete hooks', 'deterministic repeat.times', diff --git a/src/core/__tests__/dispatch-interactions.test.ts b/src/core/__tests__/dispatch-interactions.test.ts index e5f4716f1..040966472 100644 --- a/src/core/__tests__/dispatch-interactions.test.ts +++ b/src/core/__tests__/dispatch-interactions.test.ts @@ -2,6 +2,7 @@ import { test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { handleRotateGestureCommand, + handleSwipePresetCommand, handleTransformGestureCommand, } from '../dispatch-interactions.ts'; import type { Interactor } from '../interactor-types.ts'; @@ -73,6 +74,49 @@ test('handleRotateGestureCommand defaults velocity sign to match degrees', async }); }); +test('handleSwipePresetCommand resolves Android in-page swipe to content lane', async () => { + const calls: unknown[][] = []; + const interactor = { + ...makeUnusedInteractor(), + snapshot: async () => ({ + backend: 'android' as const, + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ], + }), + swipe: async (...args: unknown[]) => { + calls.push(args); + }, + }; + + const result = await handleSwipePresetCommand( + ANDROID_EMULATOR, + interactor, + ['left', '300'], + undefined, + ); + + assert.deepEqual(calls, [[360, 520, 40, 520, 300]]); + assert.deepEqual(result, { + x1: 360, + y1: 520, + x2: 40, + y2: 520, + preset: 'left', + durationMs: 300, + effectiveDurationMs: 300, + timingMode: 'direct', + count: 1, + pauseMs: 0, + pattern: 'one-way', + message: 'Swiped left', + }); +}); + test('handleRotateGestureCommand routes Android through the interactor', async () => { const calls: unknown[][] = []; const interactor = { diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index 523762dfb..9c18cac26 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -3,19 +3,17 @@ import assert from 'node:assert/strict'; import { dispatchCommand } from '../dispatch.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; -import { clearIosSimulatorAppState, openIosApp } from '../../platforms/ios/apps.ts'; +import { openIosApp, setIosSetting } from '../../platforms/ios/apps.ts'; import { openAndroidApp } from '../../platforms/android/app-lifecycle.ts'; +import { setAndroidSetting } from '../../platforms/android/settings.ts'; import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; vi.mock('../../platforms/ios/apps.ts', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - clearIosSimulatorAppState: vi.fn(async () => ({ - bundleId: 'com.example.app', - containerPath: '/tmp/com.example.app', - })), openIosApp: vi.fn(async () => {}), + setIosSetting: vi.fn(async () => ({ bundleId: 'com.example.app', cleared: true })), }; }); @@ -27,14 +25,24 @@ vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => { }; }); -const mockClearIosSimulatorAppState = vi.mocked(clearIosSimulatorAppState); +vi.mock('../../platforms/android/settings.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + setAndroidSetting: vi.fn(async () => ({ package: 'com.example.app', cleared: true })), + }; +}); + +const mockSetIosSetting = vi.mocked(setIosSetting); const mockOpenIosApp = vi.mocked(openIosApp); const mockOpenAndroidApp = vi.mocked(openAndroidApp); +const mockSetAndroidSetting = vi.mocked(setAndroidSetting); beforeEach(() => { - mockClearIosSimulatorAppState.mockClear(); + mockSetIosSetting.mockClear(); mockOpenIosApp.mockClear(); mockOpenAndroidApp.mockClear(); + mockSetAndroidSetting.mockClear(); }); test('dispatch open rejects URL as first argument when second URL is provided', async () => { @@ -120,9 +128,11 @@ test('dispatch open clears Maestro iOS simulator state and launches once', async }); assert.equal(result?.app, 'com.example.app'); - assert.equal(mockClearIosSimulatorAppState.mock.calls.length, 1); - assert.deepEqual(mockClearIosSimulatorAppState.mock.calls[0]?.slice(0, 2), [ + assert.equal(mockSetIosSetting.mock.calls.length, 1); + assert.deepEqual(mockSetIosSetting.mock.calls[0]?.slice(0, 4), [ IOS_SIMULATOR, + 'clear-app-state', + 'clear', 'com.example.app', ]); assert.equal(mockOpenIosApp.mock.calls.length, 1); @@ -133,3 +143,51 @@ test('dispatch open clears Maestro iOS simulator state and launches once', async 'true', ]); }); + +test('dispatch open clears Android app data before launch', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + const result = await dispatchCommand(device, 'open', ['com.example.app'], undefined, { + clearAppState: true, + }); + + assert.equal(result?.app, 'com.example.app'); + assert.equal(mockSetAndroidSetting.mock.calls.length, 1); + assert.deepEqual(mockSetAndroidSetting.mock.calls[0]?.slice(0, 4), [ + device, + 'clear-app-state', + 'clear', + 'com.example.app', + ]); + assert.equal(mockOpenAndroidApp.mock.calls.length, 1); +}); + +test('dispatch settings clear-app-state uses the active session app by default', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + const result = await dispatchCommand(device, 'settings', ['clear-app-state'], undefined, { + appBundleId: 'com.example.app', + }); + + assert.equal(result?.setting, 'clear-app-state'); + assert.equal(result?.state, 'clear'); + assert.equal(mockSetAndroidSetting.mock.calls.length, 1); + assert.deepEqual(mockSetAndroidSetting.mock.calls[0]?.slice(0, 4), [ + device, + 'clear-app-state', + 'clear', + 'com.example.app', + ]); +}); diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 23fe549c5..ff9fa4417 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -2,7 +2,13 @@ import { AppError } from '../utils/errors.ts'; import type { DeviceInfo } from '../utils/device.ts'; import { successText, withSuccessText } from '../utils/success-text.ts'; import { findMistargetedTypeRefToken } from '../utils/type-target-warning.ts'; -import { parseScrollDirection } from './scroll-gesture.ts'; +import { + buildSwipePresetGesturePlan, + inferGestureReferenceFrame, + parseScrollDirection, + parseSwipePreset, + type SwipePreset, +} from './scroll-gesture.ts'; import { getClickButtonValidationError, resolveClickButton, @@ -407,6 +413,58 @@ export async function handleSwipeCommand( } const requestedDurationMs = positionals[4] ? Number(positionals[4]) : 250; + return await runSwipeCoordinates({ + device, + interactor, + context, + x1, + y1, + x2, + y2, + requestedDurationMs, + }); +} + +export async function handleSwipePresetCommand( + device: DeviceInfo, + interactor: Interactor, + positionals: string[], + context: DispatchContext | undefined, +): Promise> { + const preset = parseSwipePreset(positionals[0]); + const requestedDurationMs = positionals[1] ? Number(positionals[1]) : 300; + const snapshot = await interactor.snapshot({ appBundleId: context?.appBundleId, compact: true }); + const frame = inferGestureReferenceFrame(snapshot.nodes ?? []); + if (!frame) { + throw new AppError('COMMAND_FAILED', 'Cannot infer viewport for gesture swipe preset'); + } + const plan = buildSwipePresetGesturePlan(preset, frame, { platform: device.platform }); + return await runSwipeCoordinates({ + device, + interactor, + context, + x1: plan.x1, + y1: plan.y1, + x2: plan.x2, + y2: plan.y2, + requestedDurationMs, + preset, + }); +} + +// fallow-ignore-next-line complexity +async function runSwipeCoordinates(params: { + device: DeviceInfo; + interactor: Interactor; + context: DispatchContext | undefined; + x1: number; + y1: number; + x2: number; + y2: number; + requestedDurationMs: number; + preset?: SwipePreset; +}): Promise> { + const { device, interactor, context, x1, y1, x2, y2, requestedDurationMs, preset } = params; const durationMs = requireIntInRange(requestedDurationMs, 'durationMs', 16, 10_000); const effectiveDurationMs = device.platform === 'ios' ? clampIosSwipeDuration(durationMs) : durationMs; @@ -445,6 +503,7 @@ export async function handleSwipeCommand( y1, x2, y2, + ...(preset ? { preset } : {}), durationMs, effectiveDurationMs, timingMode: 'runner-series', @@ -468,6 +527,7 @@ export async function handleSwipeCommand( y1, x2, y2, + ...(preset ? { preset } : {}), durationMs, effectiveDurationMs, timingMode: device.platform === 'ios' ? 'safe-normalized' : 'direct', @@ -475,7 +535,7 @@ export async function handleSwipeCommand( pauseMs, pattern, }, - formatSwipeMessage(count, pattern), + preset ? `Swiped ${preset}` : formatSwipeMessage(count, pattern), ); } diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 2a10c0f3b..9bc39988b 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -28,6 +28,7 @@ import { handleRotateGestureCommand, handleScrollCommand, handleSwipeCommand, + handleSwipePresetCommand, handleTransformGestureCommand, handleTypeCommand, } from './dispatch-interactions.ts'; @@ -104,6 +105,8 @@ async function dispatchKnownCommand( return await handlePressCommand(device, interactor, positionals, context); case 'swipe': return await handleSwipeCommand(device, interactor, positionals, context); + case 'swipe-preset': + return await handleSwipePresetCommand(device, interactor, positionals, context); case 'pan': return await handlePanCommand(interactor, positionals); case 'fling': @@ -224,14 +227,7 @@ async function handleOpenCommand( 'Clearing app state requires an app target, not a deep link.', ); } - if (device.platform !== 'ios' || device.kind !== 'simulator') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'Clearing app state is currently supported only on iOS simulators.', - ); - } - const { clearIosSimulatorAppState } = await import('../platforms/ios/apps.ts'); - await clearIosSimulatorAppState(device, app); + await interactor.setSetting('clear-app-state', 'clear', app); } await interactor.open(app, { activity: context?.activity, @@ -426,7 +422,31 @@ async function handleSettingsCommand( context: DispatchContext | undefined, ): Promise> { const [setting, state, target, mode] = positionals; - if (!setting || !state) { + if (!setting || (!state && setting !== 'clear-app-state')) { + throw new AppError('INVALID_ARGS', 'settings requires setting state'); + } + if (setting === 'clear-app-state') { + const appBundleId = (state === 'clear' ? target : state) ?? context?.appBundleId; + if (!appBundleId) { + throw new AppError( + 'INVALID_ARGS', + 'settings clear-app-state requires an app id or an active app session.', + ); + } + emitDiagnostic({ + level: 'debug', + phase: 'settings_apply', + data: { setting, state: 'clear', appBundleId, platform: device.platform }, + }); + const result = await interactor.setSetting(setting, 'clear', appBundleId); + return result && typeof result === 'object' + ? withSuccessText( + { setting, state: 'clear', ...result }, + readResultMessage(result) ?? `Cleared user data for ${appBundleId}`, + ) + : { setting, state: 'clear', ...successText(`Cleared user data for ${appBundleId}`) }; + } + if (!state) { throw new AppError('INVALID_ARGS', 'settings requires setting state'); } const isLocationSet = setting === 'location' && state === 'set'; diff --git a/src/core/scroll-gesture.ts b/src/core/scroll-gesture.ts index c64b32b0b..b87a4cecb 100644 --- a/src/core/scroll-gesture.ts +++ b/src/core/scroll-gesture.ts @@ -1,6 +1,8 @@ import { AppError } from '../utils/errors.ts'; +import type { Rect, SnapshotNode } from '../utils/snapshot.ts'; export type ScrollDirection = 'up' | 'down' | 'left' | 'right'; +export type SwipePreset = 'left' | 'right' | 'left-edge' | 'right-edge'; export type GestureReferenceFrame = { referenceWidth: number; @@ -38,6 +40,16 @@ export type SwipeGesturePlan = Omit & { direction: ScrollDirection; }; +export type SwipePresetGesturePlan = { + preset: SwipePreset; + x1: number; + y1: number; + x2: number; + y2: number; + referenceWidth: number; + referenceHeight: number; +}; + const DEFAULT_SCROLL_AMOUNT = 0.6; const DEFAULT_EDGE_PADDING_FRACTION = 0.05; @@ -91,6 +103,60 @@ export function buildSwipeGesturePlan(options: SwipeGestureOptions): SwipeGestur }; } +export function buildSwipePresetGesturePlan( + preset: SwipePreset, + frame: GestureReferenceFrame, + options: { platform?: string; marginPx?: number } = {}, +): SwipePresetGesturePlan { + const marginPx = options.marginPx ?? 8; + const horizontalLanePercent = options.platform === 'android' ? 65 : 50; + const [startPercent, endPercent, yPercent] = + preset === 'left' + ? [90, 10, horizontalLanePercent] + : preset === 'right' + ? [10, 90, horizontalLanePercent] + : preset === 'left-edge' + ? [99, 15, 50] + : [1, 85, 50]; + const start = pointFromPercent(frame, startPercent, yPercent, { marginPx }); + const end = pointFromPercent(frame, endPercent, yPercent, { marginPx }); + return { + preset, + x1: start.x, + y1: start.y, + x2: end.x, + y2: end.y, + referenceWidth: frame.referenceWidth, + referenceHeight: frame.referenceHeight, + }; +} + +export function parseSwipePreset(input: string | undefined): SwipePreset { + switch (input) { + case 'left': + case 'right': + case 'left-edge': + case 'right-edge': + return input; + default: + throw new AppError( + 'INVALID_ARGS', + 'gesture swipe requires left, right, left-edge, or right-edge', + ); + } +} + +export function inferGestureReferenceFrame( + nodes: Array>, +): GestureReferenceFrame | undefined { + const viewportRect = inferViewportRect(nodes); + if (!viewportRect) return undefined; + return { + referenceWidth: viewportRect.width, + referenceHeight: viewportRect.height, + }; +} + export function pointFromPercent( frame: GestureReferenceFrame, xPercent: number, @@ -141,6 +207,35 @@ function scrollDirectionForFingerSwipe(direction: ScrollDirection): ScrollDirect } } +function inferViewportRect(nodes: Array>): Rect | undefined { + const candidate = nodes + .filter((node) => isViewportNode(node.type) && isValidRect(node.rect)) + .map((node) => node.rect) + .sort( + (left, right) => + (right?.width ?? 0) * (right?.height ?? 0) - (left?.width ?? 0) * (left?.height ?? 0), + )[0]; + if (candidate) return candidate; + + const rects = nodes.map((node) => node.rect).filter(isValidRect); + if (rects.length === 0) return undefined; + + const width = Math.max(...rects.map((rect) => rect.x + rect.width)); + const height = Math.max(...rects.map((rect) => rect.y + rect.height)); + if (width <= 0 || height <= 0) return undefined; + return { x: 0, y: 0, width, height }; +} + +function isViewportNode(type: string | undefined): boolean { + if (!type) return false; + const normalized = type.toLowerCase(); + return normalized.includes('application') || normalized.includes('window'); +} + +function isValidRect(rect: Rect | undefined): rect is Rect { + return !!rect && rect.width > 0 && rect.height > 0; +} + function resolveRequestedAmount(amount: number | undefined): number { if (amount === undefined) return DEFAULT_SCROLL_AMOUNT; if (!Number.isFinite(amount) || amount <= 0) { diff --git a/src/core/settings-contract.ts b/src/core/settings-contract.ts index f3dbdeae9..f6ec74a0b 100644 --- a/src/core/settings-contract.ts +++ b/src/core/settings-contract.ts @@ -5,6 +5,7 @@ const SETTINGS_APPEARANCE_USAGE = 'appearance '; const SETTINGS_FACEID_USAGE = 'faceid '; const SETTINGS_TOUCHID_USAGE = 'touchid '; const SETTINGS_FINGERPRINT_USAGE = 'fingerprint '; +const SETTINGS_CLEAR_APP_STATE_USAGE = 'clear-app-state [app-id]'; const SETTINGS_PERMISSION_USAGE = 'permission [full|limited]'; const SETTINGS_MACOS_PERMISSION_USAGE = @@ -19,11 +20,12 @@ export const SETTINGS_USAGE_OVERRIDE = [ `settings ${SETTINGS_FACEID_USAGE}`, `settings ${SETTINGS_TOUCHID_USAGE}`, `settings ${SETTINGS_FINGERPRINT_USAGE}`, + `settings ${SETTINGS_CLEAR_APP_STATE_USAGE}`, `settings ${SETTINGS_PERMISSION_USAGE}`, `settings ${SETTINGS_MACOS_PERMISSION_USAGE}`, ].join(' | '); -export const SETTINGS_INVALID_ARGS_MESSAGE = `settings requires ${SETTINGS_WIFI_USAGE}, ${SETTINGS_LOCATION_SET_USAGE}, ${SETTINGS_ANIMATIONS_USAGE}, ${SETTINGS_APPEARANCE_USAGE}, ${SETTINGS_FACEID_USAGE}, ${SETTINGS_TOUCHID_USAGE}, ${SETTINGS_FINGERPRINT_USAGE}, ${SETTINGS_PERMISSION_USAGE}, or ${SETTINGS_MACOS_PERMISSION_USAGE}`; +export const SETTINGS_INVALID_ARGS_MESSAGE = `settings requires ${SETTINGS_WIFI_USAGE}, ${SETTINGS_LOCATION_SET_USAGE}, ${SETTINGS_ANIMATIONS_USAGE}, ${SETTINGS_APPEARANCE_USAGE}, ${SETTINGS_FACEID_USAGE}, ${SETTINGS_TOUCHID_USAGE}, ${SETTINGS_FINGERPRINT_USAGE}, ${SETTINGS_CLEAR_APP_STATE_USAGE}, ${SETTINGS_PERMISSION_USAGE}, or ${SETTINGS_MACOS_PERMISSION_USAGE}`; export function isMacOsSettingSupported(setting: string): boolean { const normalized = setting.trim().toLowerCase(); diff --git a/src/daemon/__tests__/request-handler-catalog.test.ts b/src/daemon/__tests__/request-handler-catalog.test.ts index 85920f175..84441229e 100644 --- a/src/daemon/__tests__/request-handler-catalog.test.ts +++ b/src/daemon/__tests__/request-handler-catalog.test.ts @@ -71,9 +71,7 @@ test('lease handler coverage table points at executable commands', async () => { flags: { tenant: 'tenant-a', runId: 'run-a', - ...(command === INTERNAL_COMMANDS.leaseAllocate - ? {} - : { leaseId: allocated.leaseId }), + ...(command === INTERNAL_COMMANDS.leaseAllocate ? {} : { leaseId: allocated.leaseId }), }, positionals: [], }, diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 4fb54c72a..2a94f6128 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -1519,8 +1519,8 @@ test('runReplayScriptFile resolves Maestro screen swipes from the snapshot frame assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ + ['gesture', ['swipe', 'left', '300']], ['snapshot', []], - ['swipe', ['320', '400', '80', '400', '300']], ['swipe', ['360', '400', '40', '400', '300']], ], ); @@ -1567,8 +1567,8 @@ test('runReplayScriptFile uses Android content lane for Maestro horizontal scree assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ + ['gesture', ['swipe', 'left', '300']], ['snapshot', []], - ['swipe', ['320', '520', '80', '520', '300']], ['swipe', ['360', '520', '40', '520', '300']], ], ); @@ -2042,7 +2042,13 @@ test('runReplayScriptFile writes per-action timing events to active trace', asyn sessionName: 's', logPath: path.join(root, 'log'), sessionStore, - invoke: async () => ({ ok: true, data: {} }), + invoke: async (req) => ({ + ok: true, + data: + req.command === 'click' + ? { timing: { totalDurationMs: 12, internal: { ignored: true } } } + : {}, + }), }); assert.equal(response.ok, true); @@ -2061,6 +2067,7 @@ test('runReplayScriptFile writes per-action timing events to active trace', asyn ], ); assert.equal(typeof events[1]?.durationMs, 'number'); + assert.deepEqual(events[1]?.resultTiming, { totalDurationMs: 12 }); }); test('AD_ARTIFACTS resolves to per-attempt dir when artifactsDir flag is set by the test runner', async () => { diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index 53d6713e0..96dc49c17 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -4347,8 +4347,21 @@ test('test applies per-script timeout and writes attempt artifacts', async () => const attemptDir = path.join(artifactsDir as string, 'attempt-1'); expect(fs.existsSync(path.join(attemptDir, 'replay.ad'))).toBe(true); expect(fs.existsSync(path.join(attemptDir, 'capture.png'))).toBe(true); + expect(fs.existsSync(path.join(attemptDir, 'replay-timing.ndjson'))).toBe(true); expect(fs.existsSync(path.join(attemptDir, 'result.txt'))).toBe(true); expect(fs.existsSync(path.join(attemptDir, 'failure.txt'))).toBe(true); + const timingLines = fs + .readFileSync(path.join(attemptDir, 'replay-timing.ndjson'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Record); + expect(timingLines.some((line) => line.type === 'replay_test_attempt_start')).toBe(true); + expect(timingLines.some((line) => line.type === 'replay_action_start')).toBe(true); + expect( + timingLines.some( + (line) => line.type === 'replay_test_attempt_stop' && line.timedOut === true, + ), + ).toBe(true); const resultText = fs.readFileSync(path.join(attemptDir, 'result.txt'), 'utf8'); expect(resultText).toMatch(/status: failed/); expect(resultText).toMatch(/timeoutMode: cooperative/); diff --git a/src/daemon/handlers/session-replay-action-runtime.ts b/src/daemon/handlers/session-replay-action-runtime.ts index 52ba171c7..f899e65d6 100644 --- a/src/daemon/handlers/session-replay-action-runtime.ts +++ b/src/daemon/handlers/session-replay-action-runtime.ts @@ -76,6 +76,7 @@ export async function invokeReplayAction(params: { command: resolved.command, ok: response.ok, durationMs: finishedAt - startedAt, + resultTiming: response.ok ? readResponseTiming(response.data) : undefined, errorCode: response.ok ? undefined : response.error.code, }); return response; @@ -174,6 +175,18 @@ function readReplayOutputEnv(data: unknown): Record | null { return entries.length > 0 ? Object.fromEntries(entries) : null; } +function readResponseTiming(data: unknown): Record | undefined { + if (!data || typeof data !== 'object' || Array.isArray(data)) return undefined; + const timing = (data as { timing?: unknown }).timing; + if (!timing || typeof timing !== 'object' || Array.isArray(timing)) return undefined; + return Object.fromEntries( + Object.entries(timing).filter(([, value]) => { + const kind = typeof value; + return kind === 'number' || kind === 'string' || kind === 'boolean'; + }), + ); +} + function appendReplayTraceEvent( tracePath: string | undefined, event: Record, diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 37c59e8e4..096716bf1 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -24,9 +24,10 @@ export async function runReplayScriptFile(params: { sessionName: string; logPath: string; sessionStore: SessionStore; + tracePath?: string; invoke: (req: DaemonRequest) => Promise; }): Promise { - const { req, sessionName, logPath, sessionStore, invoke } = params; + const { req, sessionName, logPath, sessionStore, tracePath, invoke } = params; const filePath = req.positionals?.[0]; if (!filePath) { return errorResponse('INVALID_ARGS', 'replay requires a path'); @@ -80,6 +81,7 @@ export async function runReplayScriptFile(params: { cliEnv: parseReplayCliEnvEntries(readReplayCliEnvEntries(req.flags?.replayEnv)), }); const shouldUpdate = req.flags?.replayUpdate === true; + const actionTracePath = tracePath ?? sessionStore.get(sessionName)?.trace?.outPath; let healed = 0; for (let index = 0; index < actions.length; index += 1) { const action = actions[index]; @@ -93,7 +95,7 @@ export async function runReplayScriptFile(params: { filePath: resolved, line: actionLines[index] ?? 0, step: index + 1, - tracePath: sessionStore.get(sessionName)?.trace?.outPath, + tracePath: actionTracePath, invoke, }); if (response.ok) { @@ -123,7 +125,7 @@ export async function runReplayScriptFile(params: { filePath: resolved, line: actionLines[index] ?? 0, step: index + 1, - tracePath: sessionStore.get(sessionName)?.trace?.outPath, + tracePath: actionTracePath, invoke, }); if (!response.ok) { diff --git a/src/daemon/handlers/session-replay.ts b/src/daemon/handlers/session-replay.ts index 298fce1ba..58567d492 100644 --- a/src/daemon/handlers/session-replay.ts +++ b/src/daemon/handlers/session-replay.ts @@ -55,6 +55,7 @@ export async function handleSessionReplayCommands(params: { requestId, artifactsDir, artifactPaths, + tracePath, }) => { const captureArtifacts = (response: DaemonResponse): DaemonResponse => { if (!artifactPaths) return response; @@ -81,6 +82,7 @@ export async function handleSessionReplayCommands(params: { sessionName: testSessionName, logPath, sessionStore, + tracePath, invoke: async (nestedReq) => captureArtifacts(await invoke(nestedReq)), }); }, diff --git a/src/daemon/handlers/session-test-runtime.ts b/src/daemon/handlers/session-test-runtime.ts index 74cc4622b..f0d4015c8 100644 --- a/src/daemon/handlers/session-test-runtime.ts +++ b/src/daemon/handlers/session-test-runtime.ts @@ -1,3 +1,5 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { normalizeError } from '../../utils/errors.ts'; @@ -42,6 +44,17 @@ export async function runReplayTestAttempt( let timeoutHandle: ReturnType | undefined; let timedOut = false; let response: DaemonResponse | undefined; + const attemptStartedAt = Date.now(); + const tracePath = prepareReplayTestTimingTrace({ + artifactsDir, + artifactPaths, + filePath, + sessionName, + requestId, + timeoutMs, + platform, + target, + }); const replayPromise = runReplay({ filePath, sessionName, @@ -50,6 +63,7 @@ export async function runReplayTestAttempt( requestId, artifactsDir, artifactPaths, + tracePath, }) .catch((error) => { const appErr = normalizeError(error); @@ -76,6 +90,15 @@ export async function runReplayTestAttempt( }), ]) : await replayPromise; + appendReplayTestTimingEvent(tracePath, { + type: 'replay_test_attempt_stop', + ts: new Date().toISOString(), + session: sessionName, + ok: response.ok, + timedOut, + durationMs: Date.now() - attemptStartedAt, + errorCode: response.ok ? undefined : response.error.code, + }); return response; } finally { if (timeoutHandle) clearTimeout(timeoutHandle); @@ -100,10 +123,31 @@ export async function runReplayTestAttempt( }); } } + const cleanupStartedAt = Date.now(); try { + appendReplayTestTimingEvent(tracePath, { + type: 'replay_test_cleanup_start', + ts: new Date().toISOString(), + session: sessionName, + }); await cleanupSession(sessionName); + appendReplayTestTimingEvent(tracePath, { + type: 'replay_test_cleanup_stop', + ts: new Date().toISOString(), + session: sessionName, + ok: true, + durationMs: Date.now() - cleanupStartedAt, + }); } catch (error) { const appErr = normalizeError(error); + appendReplayTestTimingEvent(tracePath, { + type: 'replay_test_cleanup_stop', + ts: new Date().toISOString(), + session: sessionName, + ok: false, + durationMs: Date.now() - cleanupStartedAt, + errorCode: appErr.code, + }); emitDiagnostic({ level: 'warn', phase: 'test_cleanup_failed', @@ -159,6 +203,52 @@ function markReplayTimeoutCleanupPending(response: DaemonResponse | undefined): }; } +function prepareReplayTestTimingTrace(params: { + artifactsDir?: string; + artifactPaths: Set; + filePath: string; + sessionName: string; + requestId: string; + timeoutMs?: number; + platform?: ReplayScriptMetadata['platform']; + target?: ReplayScriptMetadata['target']; +}): string | undefined { + const { + artifactsDir, + artifactPaths, + filePath, + sessionName, + requestId, + timeoutMs, + platform, + target, + } = params; + if (!artifactsDir) return undefined; + const tracePath = path.join(artifactsDir, 'replay-timing.ndjson'); + fs.mkdirSync(path.dirname(tracePath), { recursive: true }); + fs.writeFileSync(tracePath, ''); + artifactPaths.add(tracePath); + appendReplayTestTimingEvent(tracePath, { + type: 'replay_test_attempt_start', + ts: new Date().toISOString(), + replayPath: filePath, + session: sessionName, + requestId, + timeoutMs, + platform, + target, + }); + return tracePath; +} + +function appendReplayTestTimingEvent( + tracePath: string | undefined, + event: Record, +): void { + if (!tracePath) return; + fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); +} + function createReplayTestTimeoutResponse( timeoutMs: number, artifactPaths: string[] = [], diff --git a/src/daemon/handlers/session-test-types.ts b/src/daemon/handlers/session-test-types.ts index ee8964501..669fa8d6e 100644 --- a/src/daemon/handlers/session-test-types.ts +++ b/src/daemon/handlers/session-test-types.ts @@ -9,6 +9,7 @@ export type ReplayTestRunReplayParams = { requestId?: string; artifactsDir?: string; artifactPaths?: Set; + tracePath?: string; }; export type ReplayTestRunReplay = (params: ReplayTestRunReplayParams) => Promise; diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index f8c84e4fc..54e50a73c 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -29,6 +29,7 @@ import { normalizeError } from '../utils/errors.ts'; const GESTURE_PLATFORM_COMMANDS: Readonly> = { pan: 'pan', fling: 'fling', + swipe: 'swipe-preset', pinch: 'pinch', rotate: 'rotate-gesture', transform: 'transform-gesture', @@ -225,7 +226,8 @@ function resolveDispatchCommand(req: DaemonRequest): DispatchCommandResolution { ) { return { ok: false, - message: 'Use gesture pan, gesture fling, gesture rotate, or gesture transform.', + message: + 'Use gesture pan, gesture fling, gesture swipe, gesture rotate, or gesture transform.', }; } if (req.command !== 'gesture') { diff --git a/src/daemon/touch-reference-frame.ts b/src/daemon/touch-reference-frame.ts index b5b917772..ef64d1c9a 100644 --- a/src/daemon/touch-reference-frame.ts +++ b/src/daemon/touch-reference-frame.ts @@ -1,4 +1,5 @@ -import type { Rect, SnapshotNode, SnapshotState } from '../utils/snapshot.ts'; +import { inferGestureReferenceFrame } from '../core/scroll-gesture.ts'; +import type { SnapshotState } from '../utils/snapshot.ts'; export type TouchReferenceFrame = { referenceWidth: number; @@ -20,42 +21,4 @@ export function getSnapshotReferenceFrame( return inferred; } -function inferTouchReferenceFrame( - nodes: Array>, -): TouchReferenceFrame | undefined { - const viewportRect = inferViewportRect(nodes); - if (!viewportRect) return undefined; - return { - referenceWidth: viewportRect.width, - referenceHeight: viewportRect.height, - }; -} - -function inferViewportRect(nodes: Array>): Rect | undefined { - const candidate = nodes - .filter((node) => isViewportNode(node.type) && isValidRect(node.rect)) - .map((node) => node.rect) - .sort( - (left, right) => - (right?.width ?? 0) * (right?.height ?? 0) - (left?.width ?? 0) * (left?.height ?? 0), - )[0]; - if (candidate) return candidate; - - const rects = nodes.map((node) => node.rect).filter(isValidRect); - if (rects.length === 0) return undefined; - - const width = Math.max(...rects.map((rect) => rect.x + rect.width)); - const height = Math.max(...rects.map((rect) => rect.y + rect.height)); - if (width <= 0 || height <= 0) return undefined; - return { x: 0, y: 0, width, height }; -} - -function isViewportNode(type: string | undefined): boolean { - if (!type) return false; - const normalized = type.toLowerCase(); - return normalized.includes('application') || normalized.includes('window'); -} - -function isValidRect(rect: Rect | undefined): rect is Rect { - return !!rect && rect.width > 0 && rect.height > 0; -} +const inferTouchReferenceFrame = inferGestureReferenceFrame; diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index 367de7a23..fa65b2359 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -5,6 +5,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { + closeAndroidApp, inferAndroidAppName, installAndroidApp, installAndroidInstallablePath, @@ -84,6 +85,32 @@ function androidOpenAdbScript(): string { ].join('\n'); } +function androidSnapshotHelperStateFileScript(): string[] { + return [ + 'if [ "$1" = "shell" ] && [ "$2" = "cmd" ] && [ "$3" = "package" ] && [ "$4" = "list" ] && [ "$5" = "packages" ] && [ "$6" = "--show-versioncode" ] && [ "$7" = "com.callstack.agentdevice.snapshothelper" ]; then', + ' printf "package:com.callstack.agentdevice.snapshothelper versionCode:999999\\n"', + ' exit 0', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "am" ] && [ "$3" = "instrument" ]; then', + ' text="$(cat "$STATE_FILE" 2>/dev/null)"', + ' xml="$(printf "" "$text")"', + ' payload="$(printf "%s" "$xml" | base64 | tr -d "\\n")"', + ' printf "INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1\\n"', + ' printf "INSTRUMENTATION_STATUS: helperApiVersion=1\\n"', + ' printf "INSTRUMENTATION_STATUS: outputFormat=uiautomator-xml\\n"', + ' printf "INSTRUMENTATION_STATUS: chunkIndex=0\\n"', + ' printf "INSTRUMENTATION_STATUS: chunkCount=1\\n"', + ' printf "INSTRUMENTATION_STATUS: payloadBase64=%s\\n" "$payload"', + ' printf "INSTRUMENTATION_STATUS_CODE: 1\\n"', + ' printf "INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1\\n"', + ' printf "INSTRUMENTATION_RESULT: helperApiVersion=1\\n"', + ' printf "INSTRUMENTATION_RESULT: ok=true\\n"', + ' printf "INSTRUMENTATION_CODE: 0\\n"', + ' exit 0', + 'fi', + ]; +} + test('parseUiHierarchy reads double-quoted Android node attributes', () => { const xml = ''; @@ -96,6 +123,7 @@ test('parseUiHierarchy reads double-quoted Android node attributes', () => { assert.deepEqual(result.nodes[0]!.rect, { x: 10, y: 20, width: 100, height: 40 }); assert.equal(result.nodes[0]!.hittable, true); assert.equal(result.nodes[0]!.enabled, true); + assert.equal(result.nodes[0]!.visibleToUser, undefined); }); test('parseUiHierarchy reads single-quoted Android node attributes', () => { @@ -135,7 +163,7 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => { test('androidUiNodes exposes decoded Android hierarchy metadata', () => { const xml = - ''; + ''; assert.deepEqual(Array.from(androidUiNodes(xml)), [ { @@ -148,13 +176,115 @@ test('androidUiNodes exposes decoded Android hierarchy metadata', () => { rect: { x: 10, y: 20, width: 100, height: 50 }, clickable: false, enabled: true, + visibleToUser: true, focusable: true, focused: true, password: true, + windowIndex: 0, + windowType: 1, + windowLayer: 3, + windowActive: true, + windowFocused: false, + windowRect: { x: 0, y: 0, width: 390, height: 844 }, }, ]); }); +test('parseUiHierarchy discards stale inactive Android application windows', () => { + const xml = ` + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Foreground article'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Stale drawer item'), + false, + ); +}); + +test('parseUiHierarchy keeps the active Android application overlay window', () => { + const xml = ` + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Covered content'), + false, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Foreground drawer item'), + true, + ); +}); + +test('parseUiHierarchy keeps only the top active Android application window', () => { + const xml = ` + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Active stale content'), + false, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Top active content'), + true, + ); +}); + +test('parseUiHierarchy excludes Android nodes that are not visible to the user', () => { + const xml = ` + + + + +`; + + const result = parseUiHierarchy(xml, 800, { interactiveOnly: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Visible action'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Hidden drawer action'), + false, + ); +}); + +test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapshots', () => { + const xml = ` + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal(result.nodes[0]!.visibleToUser, true); + assert.equal(result.nodes[1]!.label, 'Hidden drawer action'); + assert.equal(result.nodes[1]!.visibleToUser, false); +}); + test('parseUiHierarchy ignores attribute-name prefix spoofing', () => { const xml = ""; @@ -534,6 +664,145 @@ test('openAndroidApp rejects activity override for deep link URLs', async () => ); }); +test('closeAndroidApp waits until package is no longer foreground', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + const calls: string[][] = []; + let focusPolls = 0; + + await withAndroidAdbProvider( + { + exec: async (args) => { + calls.push(args); + if (args.join(' ') === 'shell dumpsys window windows') { + focusPolls += 1; + return { + stdout: + focusPolls === 1 + ? 'mCurrentFocus=Window{42 u0 com.example.app/.MainActivity}\n' + : 'mCurrentFocus=Window{43 u0 com.android.launcher/.Launcher}\n', + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + reverse: { + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + }, + }, + { serial: 'emulator-5554' }, + async () => await closeAndroidApp(device, 'com.example.app'), + ); + + assert.deepEqual(calls, [ + ['shell', 'am', 'force-stop', 'com.example.app'], + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'pidof', 'com.example.app'], + ['shell', 'pidof', 'com.example.app'], + ]); +}); + +test('closeAndroidApp returns after force-stop when package is already not foreground', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + const calls: string[][] = []; + + await withAndroidAdbProvider( + { + exec: async (args) => { + calls.push(args); + if (args.join(' ') === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{43 u0 com.android.launcher/.Launcher}\n', + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + reverse: { + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + }, + }, + { serial: 'emulator-5554' }, + async () => await closeAndroidApp(device, 'com.example.app'), + ); + + assert.deepEqual(calls, [ + ['shell', 'am', 'force-stop', 'com.example.app'], + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'pidof', 'com.example.app'], + ['shell', 'pidof', 'com.example.app'], + ]); +}); + +test('closeAndroidApp waits until package process exits after force-stop', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + const calls: string[][] = []; + let processPolls = 0; + + await withAndroidAdbProvider( + { + exec: async (args) => { + calls.push(args); + if (args.join(' ') === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{43 u0 com.android.launcher/.Launcher}\n', + stderr: '', + exitCode: 0, + }; + } + if (args.join(' ') === 'shell pidof com.example.app') { + processPolls += 1; + return { + stdout: processPolls === 1 ? '12345\n' : '', + stderr: '', + exitCode: processPolls === 1 ? 0 : 1, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + reverse: { + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + }, + }, + { serial: 'emulator-5554' }, + async () => await closeAndroidApp(device, 'com.example.app'), + ); + + assert.deepEqual(calls, [ + ['shell', 'am', 'force-stop', 'com.example.app'], + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'pidof', 'com.example.app'], + ['shell', 'pidof', 'com.example.app'], + ['shell', 'pidof', 'com.example.app'], + ]); +}); + test('openAndroidApp ensures Android reverse before localhost deep link launch', async () => { const device: DeviceInfo = { platform: 'android', @@ -874,6 +1143,38 @@ test('rotateAndroid locks auto-rotate and sets user rotation', async () => { ); }); +test('setAndroidSetting clear-app-state force stops and clears package data', async () => { + await withMockedAdb( + 'agent-device-android-clear-app-state-', + [ + '#!/bin/sh', + 'printf "__CMD__\\n" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', + 'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', + 'if [ "$1" = "-s" ]; then', + ' shift', + ' shift', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "am" ] && [ "$3" = "force-stop" ] && [ "$4" = "com.example.app" ]; then', + ' exit 0', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "pm" ] && [ "$3" = "clear" ] && [ "$4" = "com.example.app" ]; then', + ' echo "Success"', + ' exit 0', + 'fi', + 'echo "unexpected args: $@" >&2', + 'exit 1', + '', + ].join('\n'), + async ({ argsLogPath, device }) => { + const result = await setAndroidSetting(device, 'clear-app-state', 'clear', 'com.example.app'); + assert.deepEqual(result, { package: 'com.example.app', cleared: true }); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\nam\nforce-stop\ncom\.example\.app/); + assert.match(logged, /shell\npm\nclear\ncom\.example\.app/); + }, + ); +}); + test('setAndroidSetting fingerprint retries emulator command when shell cmd fingerprint fails', async () => { await withMockedAdb( 'agent-device-android-fingerprint-fallback-', @@ -1446,6 +1747,7 @@ test('fillAndroid uses chunk-safe shell input and retries when verification stil ' shift', ' shift', 'fi', + ...androidSnapshotHelperStateFileScript(), 'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "tap" ]; then', ' exit 0', 'fi', @@ -1499,6 +1801,7 @@ test('fillAndroid keeps delayed typing in typed-input mode', async () => { ' shift', ' shift', 'fi', + ...androidSnapshotHelperStateFileScript(), 'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "tap" ]; then', ' exit 0', 'fi', diff --git a/src/platforms/android/__tests__/snapshot-helper-session.test.ts b/src/platforms/android/__tests__/snapshot-helper-session.test.ts index ca1665685..5e44b80ec 100644 --- a/src/platforms/android/__tests__/snapshot-helper-session.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-session.test.ts @@ -46,6 +46,39 @@ test('returns undefined when the adb provider cannot spawn a helper process', as assert.deepEqual(calls, []); }); +test('disables repeated persistent session attempts after startup failure', async () => { + const calls: string[][] = []; + const spawnArgs: string[][] = []; + const provider: AndroidAdbProvider = { + exec: async (args) => { + calls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + spawn: (args) => { + spawnArgs.push(args); + const process = new FakeAndroidProcess(); + queueMicrotask(() => process.emitExit(0, null)); + return process; + }, + }; + + const first = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + const second = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + + assert.equal(first, undefined); + assert.equal(second, undefined); + assert.equal(spawnArgs.length, 1); + assert.equal(calls.filter((args) => args[0] === 'forward').length, 2); +}); + test('starts and reuses a persistent Android snapshot helper session', async () => { const calls: string[][] = []; const spawnArgs: string[][] = []; @@ -74,7 +107,10 @@ test('starts and reuses a persistent Android snapshot helper session', async () assert.equal(second?.metadata.transport, 'persistent-session'); assert.equal(second?.metadata.sessionReused, true); assert.equal(spawnArgs.length, 1); - assert.equal(calls.filter((args) => args[0] === 'forward' && args[1]?.startsWith('tcp:')).length, 1); + assert.equal( + calls.filter((args) => args[0] === 'forward' && args[1]?.startsWith('tcp:')).length, + 1, + ); }); test('restarts the helper session when capture options change', async () => { @@ -97,7 +133,10 @@ test('restarts the helper session when capture options change', async () => { assert.equal(restarted?.metadata.sessionReused, false); assert.equal(spawnArgs.length, 2); - assert.equal(calls.some((args) => args[0] === 'forward' && args[1] === '--remove'), true); + assert.equal( + calls.some((args) => args[0] === 'forward' && args[1] === '--remove'), + true, + ); }); test('invalidates the helper session after a malformed response', async () => { @@ -116,7 +155,10 @@ test('invalidates the helper session after a malformed response', async () => { }, ); - assert.equal(calls.some((args) => args[0] === 'forward' && args[1] === '--remove'), true); + assert.equal( + calls.some((args) => args[0] === 'forward' && args[1] === '--remove'), + true, + ); }); function createSessionProvider(options: { diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 84bd6ae89..801dd8430 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -81,6 +81,9 @@ function createHelperAdb( ): AndroidAdbExecutor { return async (args, options) => { if (args.includes('--show-versioncode')) return installedHelperProbe; + if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop') { + return { exitCode: 0, stdout: '', stderr: '' }; + } if (args.includes('instrument') && handlers.instrument) { return await handlers.instrument(args, options); } @@ -476,6 +479,9 @@ test('snapshotAndroid falls back to stock uiautomator when helper fails', async if (args.includes('exec-out')) { return { exitCode: 0, stdout: stockXml, stderr: '' }; } + if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop') { + return { exitCode: 0, stdout: '', stderr: '' }; + } return { exitCode: 1, stdout: '', stderr: 'instrumentation failed' }; }; @@ -492,7 +498,7 @@ test('snapshotAndroid falls back to stock uiautomator when helper fails', async ); assert.deepEqual( adbCalls.map((args) => args[0]), - ['shell', 'shell', 'exec-out'], + ['shell', 'shell', 'shell', 'exec-out'], ); assert.equal(mockRunCmd.mock.calls.length, 0); }); @@ -661,10 +667,17 @@ test('snapshotAndroid skips stock fallback after killed helper instrumentation', test('snapshotAndroid falls back to stock dump after unparseable helper output', async () => { const stockXml = ''; - const helperAdb = createHelperAdb({ - instrument: async () => ({ exitCode: 0, stdout: '', stderr: '' }), - stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }), - }); + const calls: string[][] = []; + const helperAdb: AndroidAdbExecutor = async (args) => { + calls.push(args); + if (args.includes('--show-versioncode')) return installedHelperProbe; + if (args.includes('instrument')) return { exitCode: 0, stdout: '', stderr: '' }; + if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop') { + return { exitCode: 0, stdout: '', stderr: '' }; + } + if (args.includes('exec-out')) return { exitCode: 0, stdout: stockXml, stderr: '' }; + throw new Error(`unexpected helper adb args: ${args.join(' ')}`); + }; const result = await snapshotAndroidWithHelper(helperAdb); @@ -673,6 +686,21 @@ test('snapshotAndroid falls back to stock dump after unparseable helper output', result.androidSnapshot.fallbackReason ?? '', /Android snapshot helper output could not be parsed/, ); + assert.equal( + calls.some((args) => args.includes('instrument')), + true, + ); + assert.equal( + calls.some( + (args) => args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper', + ), + true, + ); + assert.equal( + calls.some((args) => args.includes('exec-out')), + true, + ); + assert.equal(mockSleep.mock.calls.at(-1)?.[0], 150); }); test('snapshotAndroid falls back to stock dump after helper adb timeout', async () => { diff --git a/src/platforms/android/app-lifecycle.ts b/src/platforms/android/app-lifecycle.ts index 77dadb9df..017be8bbe 100644 --- a/src/platforms/android/app-lifecycle.ts +++ b/src/platforms/android/app-lifecycle.ts @@ -3,6 +3,7 @@ import os from 'node:os'; import path from 'node:path'; import { resolveFileOverridePath, runCmd, whichCmd } from '../../utils/exec.ts'; import { AppError } from '../../utils/errors.ts'; +import { sleep } from '../../utils/timeouts.ts'; import type { AppsFilter } from '../../commands/app-inventory-contract.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import { isDeepLinkTarget } from '../../core/open-target.ts'; @@ -45,6 +46,11 @@ const ANDROID_APPS_DISCOVERY_HINT = const ANDROID_AMBIGUOUS_APP_HINT = 'Run agent-device apps --platform android to see the exact installed package names before retrying open.'; const ANDROID_LOCALHOST_HOSTNAMES = new Set(['localhost', '127.0.0.1', '::1', '[::1]']); +const ANDROID_CLOSE_FOCUS_TIMEOUT_MS = 2_000; +const ANDROID_CLOSE_FOCUS_POLL_MS = 50; +const ANDROID_CLOSE_PROCESS_TIMEOUT_MS = 2_000; +const ANDROID_CLOSE_PROCESS_POLL_MS = 50; +const ANDROID_CLOSE_PROCESS_GONE_STABLE_MS = 150; type AndroidAppResolution = { type: 'intent' | 'package'; value: string }; @@ -640,6 +646,7 @@ export async function closeAndroidApp(device: DeviceInfo, app: string): Promise< const trimmed = app.trim(); if (trimmed.toLowerCase() === 'settings') { await runAndroidAdb(device, ['shell', 'am', 'force-stop', 'com.android.settings']); + await waitForAndroidPackageStopped(device, 'com.android.settings'); return; } const resolved = await resolveAndroidApp(device, app); @@ -647,6 +654,65 @@ export async function closeAndroidApp(device: DeviceInfo, app: string): Promise< throw new AppError('INVALID_ARGS', 'Close requires a package name, not an intent'); } await runAndroidAdb(device, ['shell', 'am', 'force-stop', resolved.value]); + await waitForAndroidPackageStopped(device, resolved.value); +} + +async function waitForAndroidPackageStopped( + device: DeviceInfo, + packageName: string, +): Promise { + await waitForAndroidPackageNotForeground(device, packageName); + await waitForAndroidPackageProcessGone(device, packageName); +} + +async function waitForAndroidPackageNotForeground( + device: DeviceInfo, + packageName: string, +): Promise { + const deadline = Date.now() + ANDROID_CLOSE_FOCUS_TIMEOUT_MS; + while (Date.now() < deadline) { + const foreground = await readAndroidForegroundApp(device); + if (foreground?.package !== packageName) return; + await sleep(ANDROID_CLOSE_FOCUS_POLL_MS); + } +} + +async function readAndroidForegroundApp(device: DeviceInfo): Promise { + for (const args of [ + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'dumpsys', 'window'], + ['shell', 'dumpsys', 'activity', 'activities'], + ['shell', 'dumpsys', 'activity'], + ]) { + const result = await runAndroidAdb(device, args, { allowFailure: true }); + const parsed = parseAndroidForegroundApp(result.stdout ?? ''); + if (parsed) return parsed; + } + return null; +} + +async function waitForAndroidPackageProcessGone( + device: DeviceInfo, + packageName: string, +): Promise { + const deadline = Date.now() + ANDROID_CLOSE_PROCESS_TIMEOUT_MS; + while (Date.now() < deadline) { + if (!(await isAndroidPackageProcessRunning(device, packageName))) { + await sleep(ANDROID_CLOSE_PROCESS_GONE_STABLE_MS); + if (!(await isAndroidPackageProcessRunning(device, packageName))) return; + } + await sleep(ANDROID_CLOSE_PROCESS_POLL_MS); + } +} + +async function isAndroidPackageProcessRunning( + device: DeviceInfo, + packageName: string, +): Promise { + const result = await runAndroidAdb(device, ['shell', 'pidof', packageName], { + allowFailure: true, + }); + return (result.stdout ?? '').trim().length > 0; } async function uninstallAndroidApp(device: DeviceInfo, app: string): Promise<{ package: string }> { diff --git a/src/platforms/android/device-input-state.ts b/src/platforms/android/device-input-state.ts index 3f57675b4..5081cbd03 100644 --- a/src/platforms/android/device-input-state.ts +++ b/src/platforms/android/device-input-state.ts @@ -157,7 +157,11 @@ function parseAndroidKeyboardState(stdout: string): AndroidKeyboardState { focusedResourceId, inputMethodPackage, ); - emitAndroidInputOwnershipFallbackDiagnostic(focusedPackage, focusedResourceId, inputMethodPackage); + emitAndroidInputOwnershipFallbackDiagnostic( + focusedPackage, + focusedResourceId, + inputMethodPackage, + ); return { visible: visible ?? false, diff --git a/src/platforms/android/settings.ts b/src/platforms/android/settings.ts index ff131d7a4..9a75e0005 100644 --- a/src/platforms/android/settings.ts +++ b/src/platforms/android/settings.ts @@ -13,6 +13,7 @@ import { import { parseAppearanceAction } from '../appearance.ts'; import { parseSettingState } from '../setting-state.ts'; import { runAndroidAdb } from './adb.ts'; +import { resolveAndroidApp } from './app-lifecycle.ts'; const ANDROID_ANIMATION_SCALE_SETTINGS = [ 'window_animation_scale', @@ -20,6 +21,7 @@ const ANDROID_ANIMATION_SCALE_SETTINGS = [ 'animator_duration_scale', ] as const; +// fallow-ignore-next-line complexity export async function setAndroidSetting( device: DeviceInfo, setting: string, @@ -91,6 +93,43 @@ export async function setAndroidSetting( ]); return; } + case 'clear-app-state': { + if (state.toLowerCase() !== 'clear') { + throw new AppError('INVALID_ARGS', 'settings clear-app-state only supports clear.'); + } + if (!appPackage) { + throw new AppError( + 'INVALID_ARGS', + 'settings clear-app-state requires an app id or an active app session.', + ); + } + const resolved = await resolveAndroidApp(device, appPackage); + if (resolved.type === 'intent') { + throw new AppError( + 'INVALID_ARGS', + 'settings clear-app-state requires a package name, not an intent.', + ); + } + await runAndroidAdb(device, ['shell', 'am', 'force-stop', resolved.value], { + allowFailure: true, + }); + const result = await runAndroidAdb(device, ['shell', 'pm', 'clear', resolved.value], { + allowFailure: true, + }); + if (result.exitCode !== 0 || !/\bSuccess\b/i.test(result.stdout)) { + throw new AppError( + 'COMMAND_FAILED', + `Failed to clear Android app data for ${resolved.value}`, + { + package: resolved.value, + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }, + ); + } + return { package: resolved.value, cleared: true }; + } case 'fingerprint': { const action = parseAndroidFingerprintAction(state); await runAndroidFingerprintCommand(device, action); diff --git a/src/platforms/android/snapshot-helper-session.ts b/src/platforms/android/snapshot-helper-session.ts index cb136932b..4398da520 100644 --- a/src/platforms/android/snapshot-helper-session.ts +++ b/src/platforms/android/snapshot-helper-session.ts @@ -20,6 +20,7 @@ import { const SESSION_READY_TIMEOUT_MS = 10_000; const SESSION_STOP_TIMEOUT_MS = 1_000; +const SESSION_PROCESS_EXIT_TIMEOUT_MS = 2_000; const FORWARD_TIMEOUT_MS = 5_000; type AndroidSnapshotHelperSession = { @@ -33,6 +34,7 @@ type AndroidSnapshotHelperSession = { }; const sessions = new Map(); +const disabledSessionIdentities = new Map(); export async function captureAndroidSnapshotWithHelperSession( options: AndroidSnapshotHelperCaptureOptions, @@ -43,18 +45,34 @@ export async function captureAndroidSnapshotWithHelperSession( const resolved = resolveAndroidSnapshotHelperCaptureOptions(options); const deviceKey = options.deviceKey ?? 'android:default'; const identity = createSessionIdentity(deviceKey, resolved, options); + if (disabledSessionIdentities.get(deviceKey) === identity) { + return undefined; + } let session = sessions.get(deviceKey); if (session && session.identity !== identity) { await stopAndroidSnapshotHelperSession(deviceKey); session = undefined; } if (!session) { - session = await startAndroidSnapshotHelperSession({ - deviceKey, - identity, - options, - resolved, - }); + try { + session = await startAndroidSnapshotHelperSession({ + deviceKey, + identity, + options, + resolved, + }); + } catch (error) { + disabledSessionIdentities.set(deviceKey, identity); + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_session_disabled', + data: { + deviceKey, + reason: error instanceof Error ? error.message : String(error), + }, + }); + return undefined; + } } try { const reused = session.capturedCount > 0; @@ -88,6 +106,7 @@ export async function stopAndroidSnapshotHelperSession(deviceKey: string): Promi } catch { // Best effort. A completed instrumentation process can reject/ignore kill. } + await waitForProcessExit(session.process, SESSION_PROCESS_EXIT_TIMEOUT_MS); try { await removeForward(session); } catch { @@ -105,7 +124,10 @@ export async function stopAndroidSnapshotHelperSession(deviceKey: string): Promi } export async function resetAndroidSnapshotHelperSessions(): Promise { - await Promise.all([...sessions.keys()].map((deviceKey) => stopAndroidSnapshotHelperSession(deviceKey))); + await Promise.all( + [...sessions.keys()].map((deviceKey) => stopAndroidSnapshotHelperSession(deviceKey)), + ); + disabledSessionIdentities.clear(); } async function startAndroidSnapshotHelperSession(params: { @@ -128,13 +150,7 @@ async function startAndroidSnapshotHelperSession(params: { if (!runner) { throw new AppError('INVALID_ARGS', 'Android snapshot helper runner was not resolved'); } - const sessionArgs = [ - ...args.slice(0, -1), - '-e', - 'sessionPort', - String(port), - runner, - ]; + const sessionArgs = [...args.slice(0, -1), '-e', 'sessionPort', String(port), runner]; const process = params.options.adbProvider!.spawn!(sessionArgs, { allowFailure: true, captureOutput: false, @@ -168,10 +184,25 @@ async function startAndroidSnapshotHelperSession(params: { } catch { // Best effort after startup failure. } + await waitForProcessExit(process, SESSION_PROCESS_EXIT_TIMEOUT_MS); throw error; } } +function waitForProcessExit(process: AndroidAdbProcess, timeoutMs: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, timeoutMs); + process.once('close', () => { + clearTimeout(timer); + resolve(); + }); + process.once('exit', () => { + clearTimeout(timer); + resolve(); + }); + }); +} + function waitForSessionReady(process: AndroidAdbProcess, timeoutMs: number): Promise { return new Promise((resolve, reject) => { let output = ''; @@ -270,9 +301,13 @@ function parseSessionSnapshotResponse( function splitSessionResponse(response: string): { headers: Record; xml: string } { const separator = response.indexOf('\n\n'); if (separator < 0) { - throw new AppError('COMMAND_FAILED', 'Android snapshot helper session returned malformed output', { - response, - }); + throw new AppError( + 'COMMAND_FAILED', + 'Android snapshot helper session returned malformed output', + { + response, + }, + ); } return { headers: parseSessionHeaders(response.slice(0, separator)), @@ -282,9 +317,13 @@ function splitSessionResponse(response: string): { headers: Record, requestId: string): void { if (headers.agentDeviceProtocol !== ANDROID_SNAPSHOT_HELPER_PROTOCOL) { - throw new AppError('COMMAND_FAILED', 'Android snapshot helper session returned wrong protocol', { - headers, - }); + throw new AppError( + 'COMMAND_FAILED', + 'Android snapshot helper session returned wrong protocol', + { + headers, + }, + ); } if (headers.outputFormat !== ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT) { throw new AppError( @@ -311,11 +350,10 @@ function validateSessionHeaders(headers: Record, requestId: stri function validateSessionXml(headers: Record, xml: string): void { const byteLength = readAndroidSnapshotHelperMetadataNumber(headers.byteLength); if (byteLength !== undefined && Buffer.byteLength(xml, 'utf8') !== byteLength) { - throw new AppError( - 'COMMAND_FAILED', - 'Android snapshot helper session returned truncated XML', - { headers, actualByteLength: Buffer.byteLength(xml, 'utf8') }, - ); + throw new AppError('COMMAND_FAILED', 'Android snapshot helper session returned truncated XML', { + headers, + actualByteLength: Buffer.byteLength(xml, 'utf8'), + }); } if (!xml.includes('')) { throw new AppError('COMMAND_FAILED', 'Android snapshot helper session did not return XML', { diff --git a/src/platforms/android/snapshot-helper-types.ts b/src/platforms/android/snapshot-helper-types.ts index 2c356c753..3ab5dabdf 100644 --- a/src/platforms/android/snapshot-helper-types.ts +++ b/src/platforms/android/snapshot-helper-types.ts @@ -9,8 +9,8 @@ export const ANDROID_SNAPSHOT_HELPER_RUNNER = 'com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation'; export const ANDROID_SNAPSHOT_HELPER_PROTOCOL = 'android-snapshot-helper-v1'; export const ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT = 'uiautomator-xml'; -export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS = 25; -export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS = 25; +export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS = 500; +export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS = 100; export const ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS = 5_000; export type { AndroidAdbExecutor } from './adb-executor.ts'; diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index 698f27904..15e82ee96 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -26,6 +26,7 @@ import { resolveAndroidAdbProvider, type AndroidAdbProvider, } from './adb-executor.ts'; +import { sleep } from './adb.ts'; import { deriveAndroidScrollableContentHints } from './scroll-hints.ts'; import { captureAndroidSnapshotWithHelper, @@ -50,6 +51,8 @@ const UI_HIERARCHY_DUMP_TIMEOUT_MS = 8_000; const HELPER_INSTALL_TIMEOUT_MS = 30_000; const HELPER_CAPTURE_TIMEOUT_MS = 5_000; const HELPER_COMMAND_TIMEOUT_MS = 30_000; +const HELPER_RUNTIME_RESET_DELAY_MS = 150; +const HELPER_RUNTIME_RESET_TIMEOUT_MS = 2_000; const RETRYABLE_ADB_STDERR_PATTERNS = [ 'device offline', 'device not found', @@ -92,7 +95,12 @@ export async function snapshotAndroid( if (!options.interactiveOnly) { const parsed = parseUiHierarchy(xml, ANDROID_SNAPSHOT_MAX_NODES, options); if (includeHiddenContentHints) { - const nativeHints = await deriveScrollableContentHintsIfNeeded(device, parsed.nodes, xml, adb); + const nativeHints = await deriveScrollableContentHintsIfNeeded( + device, + parsed.nodes, + xml, + adb, + ); applyHiddenContentHintsToNodes(nativeHints, parsed.nodes); } return { ...parsed, androidSnapshot: capture.metadata }; @@ -139,13 +147,11 @@ async function applyHiddenContentHintsToInteractiveSnapshot(params: { params.xml, params.adb, ); - applyHiddenContentHintsToInteractiveNodes( - nativeHints, - fullSnapshot, - params.interactiveSnapshot, - ); + applyHiddenContentHintsToInteractiveNodes(nativeHints, fullSnapshot, params.interactiveSnapshot); if (nativeHints.size === 0) { - const presentationHints = deriveMobileSnapshotHiddenContentHints(attachRefs(fullSnapshot.nodes)); + const presentationHints = deriveMobileSnapshotHiddenContentHints( + attachRefs(fullSnapshot.nodes), + ); applyHiddenContentHintsToInteractiveNodes( presentationHints, fullSnapshot, @@ -287,6 +293,7 @@ async function captureAndroidUiHierarchyFromHelper( phase: 'android_snapshot_helper_session_fallback', data: { reason: normalizeError(error).message }, }); + await resetAndroidSnapshotHelperRuntime(adb, artifact.manifest.packageName); } return await withDiagnosticTimer( 'android_snapshot_helper_capture', @@ -345,6 +352,7 @@ async function recoverAndroidHelperCaptureFailure(params: { data: { reason: fallbackReason }, }); await stopAndroidSnapshotHelperSession(params.helperDeviceKey); + await resetAndroidSnapshotHelperRuntime(params.adb, params.artifact.manifest.packageName); forgetAndroidSnapshotHelperInstall({ deviceKey: params.helperDeviceKey, packageName: params.artifact.manifest.packageName, @@ -353,6 +361,30 @@ async function recoverAndroidHelperCaptureFailure(params: { return await captureStockUiHierarchy(params.device, fallbackReason, params.adb); } +async function resetAndroidSnapshotHelperRuntime( + adb: AndroidAdbExecutor, + packageName: string, +): Promise { + try { + await adb(['shell', 'am', 'force-stop', packageName], { + allowFailure: true, + timeoutMs: HELPER_RUNTIME_RESET_TIMEOUT_MS, + }); + await sleep(HELPER_RUNTIME_RESET_DELAY_MS); + emitDiagnostic({ + level: 'debug', + phase: 'android_snapshot_helper_runtime_reset', + data: { packageName }, + }); + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_runtime_reset_failed', + data: { packageName, error: normalizeError(error).message }, + }); + } +} + function formatAndroidSnapshotHelperFallbackReason(error: unknown): string { const normalized = normalizeError(error); const helperMessage = readHelperMessage(normalized.details?.helper); diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index 5bbdddaaf..c7eb18c08 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -16,12 +16,19 @@ export type AndroidUiNodeMetadata = { rect?: Rect; clickable?: boolean; enabled?: boolean; + visibleToUser?: boolean; focusable?: boolean; focused?: boolean; password?: boolean; scrollable?: boolean; canScrollForward?: boolean; canScrollBackward?: boolean; + windowIndex?: number; + windowType?: number; + windowLayer?: number; + windowActive?: boolean; + windowFocused?: boolean; + windowRect?: Rect; }; export function* androidUiNodes(xml: string): IterableIterator { @@ -163,6 +170,7 @@ function appendAndroidSnapshotNode( bundleId: node.packageName ?? undefined, rect: node.rect, enabled: node.enabled, + visibleToUser: node.visibleToUser, hittable: node.hittable, depth, parentIndex, @@ -176,7 +184,10 @@ function hasInteractiveDescendant(state: AndroidSnapshotBuildState, node: Androi const cached = state.interactiveDescendantMemo.get(node); if (cached !== undefined) return cached; for (const child of node.children) { - if (child.hittable || hasInteractiveDescendant(state, child)) { + if ( + child.visibleToUser !== false && + (child.hittable || hasInteractiveDescendant(state, child)) + ) { state.interactiveDescendantMemo.set(node, true); return true; } @@ -193,6 +204,26 @@ function readNodeAttributes(node: string): Omit { if (raw === null) return undefined; return raw === 'true'; }; + const numberAttr = (name: string): number | undefined => { + const raw = getAttr(name); + if (raw === null || raw.trim() === '') return undefined; + const value = Number(raw); + return Number.isFinite(value) ? value : undefined; + }; + const optionalNumberAttr = ( + key: Key, + name: string, + ): Pick | {} => { + const value = numberAttr(name); + return value === undefined ? {} : { [key]: value }; + }; + const optionalRectAttr = ( + key: Key, + name: string, + ): Pick | {} => { + const value = parseBounds(getAttr(name)); + return value === undefined ? {} : { [key]: value }; + }; const optionalBoolAttr = ( key: Key, name: string, @@ -212,9 +243,16 @@ function readNodeAttributes(node: string): Omit { focusable: boolAttr('focusable'), focused: boolAttr('focused'), password: boolAttr('password'), + ...optionalBoolAttr('visibleToUser', 'visible-to-user'), ...optionalBoolAttr('scrollable', 'scrollable'), ...optionalBoolAttr('canScrollForward', 'can-scroll-forward'), ...optionalBoolAttr('canScrollBackward', 'can-scroll-backward'), + ...optionalNumberAttr('windowIndex', 'window-index'), + ...optionalNumberAttr('windowType', 'window-type'), + ...optionalNumberAttr('windowLayer', 'window-layer'), + ...optionalBoolAttr('windowActive', 'window-active'), + ...optionalBoolAttr('windowFocused', 'window-focused'), + ...optionalRectAttr('windowRect', 'window-bounds'), }; } @@ -362,6 +400,7 @@ export type AndroidUiHierarchy = { packageName: string | null; rect?: Rect; enabled?: boolean; + visibleToUser?: boolean; hittable?: boolean; depth: number; parentIndex?: number; @@ -370,6 +409,12 @@ export type AndroidUiHierarchy = { scrollable?: boolean; canScrollForward?: boolean; canScrollBackward?: boolean; + windowIndex?: number; + windowType?: number; + windowLayer?: number; + windowActive?: boolean; + windowFocused?: boolean; + windowRect?: Rect; children: AndroidNode[]; }; @@ -383,6 +428,8 @@ type AndroidNodeInclusionInfo = { isVisual: boolean; }; +const ANDROID_WINDOW_TYPE_APPLICATION = 1; + export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { const root: AndroidUiHierarchy = { type: null, @@ -413,10 +460,17 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { packageName: attrs.packageName, rect: attrs.rect, enabled: attrs.enabled, + visibleToUser: attrs.visibleToUser, hittable: attrs.clickable ?? attrs.focusable, scrollable: attrs.scrollable, canScrollForward: attrs.canScrollForward, canScrollBackward: attrs.canScrollBackward, + windowIndex: attrs.windowIndex, + windowType: attrs.windowType, + windowLayer: attrs.windowLayer, + windowActive: attrs.windowActive, + windowFocused: attrs.windowFocused, + windowRect: attrs.windowRect, depth: parent.depth + 1, parentIndex: undefined, children: [], @@ -427,6 +481,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { } match = tokenRegex.exec(xml); } + discardInactiveAndroidApplicationWindows(root); applyAndroidScrollActionHints(root); return root; } @@ -442,6 +497,45 @@ function applyAndroidScrollActionHints(root: AndroidUiHierarchy): void { } } +function discardInactiveAndroidApplicationWindows(root: AndroidUiHierarchy): void { + const windows = root.children.filter(isAndroidWindowRoot); + if (windows.length < 2) return; + + // Android can keep stale application windows in the accessibility tree after drawer and + // navigation transitions. Keep dialogs/system windows, but expose only the foreground + // application layer so agents do not act on content that is hidden from users. + const foregroundApplicationWindows = windows.filter( + (window) => isAndroidApplicationWindow(window) && isAndroidForegroundWindow(window), + ); + if (foregroundApplicationWindows.length === 0) return; + const foregroundLayer = highestAndroidWindowLayer(foregroundApplicationWindows); + + root.children = root.children.filter((window) => { + if (!isAndroidApplicationWindow(window)) return true; + if (!isAndroidForegroundWindow(window)) return false; + return foregroundLayer === undefined || window.windowLayer === foregroundLayer; + }); +} + +function highestAndroidWindowLayer(windows: AndroidNode[]): number | undefined { + const layers = windows + .map((window) => window.windowLayer) + .filter((layer): layer is number => layer !== undefined); + return layers.length > 0 ? Math.max(...layers) : undefined; +} + +function isAndroidWindowRoot(node: AndroidNode): boolean { + return node.windowIndex !== undefined || node.windowType !== undefined; +} + +function isAndroidApplicationWindow(node: AndroidNode): boolean { + return node.windowType === ANDROID_WINDOW_TYPE_APPLICATION; +} + +function isAndroidForegroundWindow(node: AndroidNode): boolean { + return node.windowActive === true || node.windowFocused === true; +} + function isVerticalScrollableNode(node: AndroidNode): boolean { if (!node.scrollable || !isScrollableType(node.type)) return false; const type = `${node.type ?? ''}`.toLowerCase(); @@ -474,6 +568,7 @@ function shouldIncludeAndroidNode( descendantHittable: boolean, ancestorCollection: boolean, ): boolean { + if (node.visibleToUser === false) return false; const info = getAndroidNodeInclusionInfo(node); if (options.interactiveOnly) { return shouldIncludeInteractiveAndroidNode( diff --git a/src/platforms/ios/__tests__/index.test.ts b/src/platforms/ios/__tests__/index.test.ts index 78af6fdf0..1b218dbc9 100644 --- a/src/platforms/ios/__tests__/index.test.ts +++ b/src/platforms/ios/__tests__/index.test.ts @@ -2403,6 +2403,56 @@ exit 1 ); }); +test('setIosSetting clear-app-state wipes iOS simulator app data container', async () => { + await withMockedXcrun( + 'agent-device-ios-clear-app-state-test-', + `#!/bin/sh +printf "__CMD__\\n" >> "$AGENT_DEVICE_TEST_ARGS_FILE" +printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE" +if [ "$1" = "simctl" ] && [ "$2" = "list" ] && [ "$3" = "devices" ] && [ "$4" = "-j" ]; then + cat <<'JSON' +{"devices":{"com.apple.CoreSimulator.SimRuntime.iOS-18-0":[{"udid":"sim-1","state":"Booted"}]}} +JSON + exit 0 +fi +if [ "$1" = "simctl" ] && [ "$2" = "terminate" ] && [ "$3" = "sim-1" ] && [ "$4" = "com.example.app" ]; then + exit 0 +fi +if [ "$1" = "simctl" ] && [ "$2" = "get_app_container" ] && [ "$3" = "sim-1" ] && [ "$4" = "com.example.app" ] && [ "$5" = "data" ]; then + echo "$AGENT_DEVICE_TEST_CONTAINER" + exit 0 +fi +echo "unexpected xcrun args: $@" >&2 +exit 1 +`, + async ({ tmpDir, argsLogPath }) => { + const containerPath = path.join(tmpDir, 'container'); + await fs.mkdir(path.join(containerPath, 'Documents'), { recursive: true }); + await fs.writeFile(path.join(containerPath, 'Documents', 'db.sqlite'), 'db'); + await fs.writeFile(path.join(containerPath, 'Library.plist'), 'prefs'); + const previousContainer = process.env.AGENT_DEVICE_TEST_CONTAINER; + process.env.AGENT_DEVICE_TEST_CONTAINER = containerPath; + try { + const result = await setIosSetting( + IOS_TEST_SIMULATOR, + 'clear-app-state', + 'clear', + 'com.example.app', + ); + assert.equal(result?.cleared, true); + assert.equal(result?.bundleId, 'com.example.app'); + assert.deepEqual(await fs.readdir(containerPath), []); + } finally { + if (previousContainer === undefined) delete process.env.AGENT_DEVICE_TEST_CONTAINER; + else process.env.AGENT_DEVICE_TEST_CONTAINER = previousContainer; + } + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /simctl\nterminate\nsim-1\ncom\.example\.app/); + assert.match(logged, /simctl\nget_app_container\nsim-1\ncom\.example\.app\ndata/); + }, + ); +}); + test('setIosSetting permission grant photos limited maps to photos-add', async () => { await withMockedXcrun( 'agent-device-ios-permission-photos-test-', diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index d8d16a192..382e2994e 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -196,11 +196,7 @@ export async function openIosApp( throw new AppError('INVALID_ARGS', 'open requires a valid URL target'); } if (device.kind === 'simulator') { - if (launchArgs && launchArgs.length > 0) { - throw new AppError('INVALID_ARGS', IOS_SIMULATOR_LAUNCH_ARGS_WITH_URL_MESSAGE); - } - await ensureBootedSimulator(device); - await runSimctl(device, ['openurl', device.id, explicitUrl]); + await openIosSimulatorUrl(device, explicitUrl, launchArgs); return; } const appBundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); @@ -221,11 +217,7 @@ export async function openIosApp( throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } if (device.kind === 'simulator') { - if (launchArgs && launchArgs.length > 0) { - throw new AppError('INVALID_ARGS', IOS_SIMULATOR_LAUNCH_ARGS_WITH_URL_MESSAGE); - } - await ensureBootedSimulator(device); - await runSimctl(device, ['openurl', device.id, deepLinkTarget]); + await openIosSimulatorUrl(device, deepLinkTarget, launchArgs); return; } const bundleId = resolveIosDeviceDeepLinkBundleId(options?.appBundleId, deepLinkTarget); @@ -251,6 +243,18 @@ export async function openIosApp( await launchIosDeviceProcess(device, bundleId, { launchArgs }); } +async function openIosSimulatorUrl( + device: DeviceInfo, + url: string, + launchArgs: string[] | undefined, +): Promise { + if (launchArgs && launchArgs.length > 0) { + throw new AppError('INVALID_ARGS', IOS_SIMULATOR_LAUNCH_ARGS_WITH_URL_MESSAGE); + } + await ensureBootedSimulator(device); + await runSimctl(device, ['openurl', device.id, url]); +} + export async function openIosDevice(device: DeviceInfo): Promise { if (device.platform === 'macos') { return; @@ -294,7 +298,7 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { @@ -530,6 +534,19 @@ export async function setIosSetting( const normalized = setting.toLowerCase(); switch (normalized) { + case 'clear-app-state': { + if (state.toLowerCase() !== 'clear') { + throw new AppError('INVALID_ARGS', 'settings clear-app-state only supports clear.'); + } + if (!appBundleId) { + throw new AppError( + 'INVALID_ARGS', + 'settings clear-app-state requires an app id or an active app session.', + ); + } + const result = await clearIosSimulatorAppState(device, appBundleId); + return { bundleId: result.bundleId, containerPath: result.containerPath, cleared: true }; + } case 'wifi': { const enabled = parseSettingState(state); const mode = enabled ? 'active' : 'failed'; diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts index ac72b639b..f3cfbcab4 100644 --- a/src/replay/__tests__/script.test.ts +++ b/src/replay/__tests__/script.test.ts @@ -107,12 +107,13 @@ test('snapshot replay script parses full refresh flags', () => { assert.equal(parsed[0]?.flags.snapshotScope, '@e1'); }); -test('gesture replay script parses pan, fling, pinch, and rotate gesture commands', () => { +test('gesture replay script parses pan, fling, swipe, pinch, and rotate gesture commands', () => { const parsed = parseReplayScript( [ 'gesture pan 195 443 80 0', 'wait "pan changed yes" 5000', 'gesture fling right 195 443 180', + 'gesture swipe right-edge 300', 'gesture pinch 1.25 195 443', 'gesture rotate 35 195 443', '', @@ -121,12 +122,13 @@ test('gesture replay script parses pan, fling, pinch, and rotate gesture command assert.deepEqual( parsed.map((action) => action.command), - ['gesture', 'wait', 'gesture', 'gesture', 'gesture'], + ['gesture', 'wait', 'gesture', 'gesture', 'gesture', 'gesture'], ); assert.deepEqual(parsed[0]?.positionals, ['pan', '195', '443', '80', '0']); assert.deepEqual(parsed[2]?.positionals, ['fling', 'right', '195', '443', '180']); - assert.deepEqual(parsed[3]?.positionals, ['pinch', '1.25', '195', '443']); - assert.deepEqual(parsed[4]?.positionals, ['rotate', '35', '195', '443']); + assert.deepEqual(parsed[3]?.positionals, ['swipe', 'right-edge', '300']); + assert.deepEqual(parsed[4]?.positionals, ['pinch', '1.25', '195', '443']); + assert.deepEqual(parsed[5]?.positionals, ['rotate', '35', '195', '443']); }); test('type and fill replay scripts round-trip typing delay flags', () => { diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index a3bfa03f8..e6d0d70e2 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -900,7 +900,7 @@ test('usage includes concise top-level commands', () => { assert.match(usageText, /clipboard read \| clipboard write /); assert.match(usageText, /keyboard \[action\]/); assert.match(usageText, /trigger-app-event \[payloadJson\]/); - assert.match(usageText, /gesture \.\.\./); + assert.match(usageText, /gesture \.\.\./); assert.doesNotMatch(usageText, /^ pan \[durationMs\]/m); assert.doesNotMatch(usageText, /^ fling /m); assert.doesNotMatch(usageText, /^ pinch \[x\] \[y\]/m); @@ -1552,6 +1552,7 @@ test('settings usage documents canonical faceid states', () => { const help = usageForCommand('settings'); if (help === null) throw new Error('Expected command help text'); assert.match(help, /location set /); + assert.match(help, /clear-app-state \[app-id\]/); assert.match(help, /light\|dark\|toggle/); assert.match(help, /match\|nonmatch\|enroll\|unenroll/); assert.match( diff --git a/src/utils/__tests__/selector-is-predicates.test.ts b/src/utils/__tests__/selector-is-predicates.test.ts index b723cce1f..8a7a7ee15 100644 --- a/src/utils/__tests__/selector-is-predicates.test.ts +++ b/src/utils/__tests__/selector-is-predicates.test.ts @@ -97,6 +97,29 @@ test('visible predicate uses visible Android ancestor geometry for rectless text assert.equal(result.pass, true); }); +test('visible predicate treats Android nodes hidden from users as hidden', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.Button', + label: 'Drawer item', + rect: { x: 0, y: 0, width: 200, height: 80 }, + hittable: true, + visibleToUser: false, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[0]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, false); +}); + test('visible predicate does not use non-hittable Android layout ancestors for rectless text', () => { const nodes: SnapshotNode[] = [ { diff --git a/src/utils/cli-command-overrides.ts b/src/utils/cli-command-overrides.ts index 13afdfa6e..020523fe2 100644 --- a/src/utils/cli-command-overrides.ts +++ b/src/utils/cli-command-overrides.ts @@ -242,12 +242,12 @@ const CLI_COMMAND_OVERRIDES = { allowedFlags: ['count', 'pauseMs', 'pattern'], }, gesture: { - usageOverride: 'gesture ...', - listUsageOverride: 'gesture ...', + usageOverride: 'gesture ...', + listUsageOverride: 'gesture ...', helpDescription: - 'Run touch gestures: pan [durationMs], fling [distance] [durationMs], pinch [x] [y], rotate [x] [y] [velocity], or transform [durationMs]', - summary: 'Run pan, fling, pinch, rotate, or transform gestures', - positionalArgs: ['pan|fling|pinch|rotate|transform', 'args?'], + 'Run touch gestures: pan [durationMs], fling [distance] [durationMs], swipe [durationMs], pinch [x] [y], rotate [x] [y] [velocity], or transform [durationMs]', + summary: 'Run pan, fling, swipe, pinch, rotate, or transform gestures', + positionalArgs: ['pan|fling|swipe|pinch|rotate|transform', 'args?'], allowsExtraPositionals: true, }, focus: { diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 8c0864ab6..5fe27c9fe 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -104,7 +104,7 @@ Command shape: Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output. If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @, @ref, @Label_Name, or @eN placeholders. Close means agent-device close. App-owned back means back; system back means back --system. - Taps are press or click. Gestures use swipe, longpress, or gesture . Android pinch, rotate, and transform use provider-native touch injection when available, then the bundled multi-touch helper. iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path; otherwise it reports UNSUPPORTED_OPERATION. + Taps are press or click. Gestures use swipe, longpress, or gesture . Use gesture swipe left|right for reliable in-page horizontal swipes, and gesture swipe right-edge for left-edge navigation/back gestures. Android pinch, rotate, and transform use provider-native touch injection when available, then the bundled multi-touch helper. iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path; otherwise it reports UNSUPPORTED_OPERATION. Bootstrap: agent-device devices --platform ios diff --git a/src/utils/selector-is-predicates.ts b/src/utils/selector-is-predicates.ts index 37e00d72e..bc58c4bf9 100644 --- a/src/utils/selector-is-predicates.ts +++ b/src/utils/selector-is-predicates.ts @@ -62,6 +62,7 @@ function isAssertionVisible( nodes: SnapshotState['nodes'], platform: Platform, ): boolean { + if (platform === 'android' && node.visibleToUser === false) return false; if (hasPositiveRect(node.rect)) return isRectVisibleInViewport(node, nodes); if (node.rect) return false; if (platform !== 'android' && node.hittable === true) return true; @@ -89,10 +90,12 @@ function resolveVisibilityAnchor( ); } +// fallow-ignore-next-line complexity function isUsefulVisibilityAnchor( node: SnapshotState['nodes'][number], platform: Platform, ): boolean { + if (platform === 'android' && node.visibleToUser === false) return false; const type = normalizeType(node.type ?? ''); // These containers often report the full content frame, not the clipped on-screen geometry. if ( diff --git a/src/utils/snapshot.ts b/src/utils/snapshot.ts index c7763af55..4ba2b9acd 100644 --- a/src/utils/snapshot.ts +++ b/src/utils/snapshot.ts @@ -38,6 +38,7 @@ export type RawSnapshotNode = { enabled?: boolean; selected?: boolean; focused?: boolean; + visibleToUser?: boolean; hittable?: boolean; depth?: number; parentIndex?: number; diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 219c273ca..327f57ae5 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1291,7 +1291,17 @@ function assertAndroidSettingsContract(world: AndroidSettingsWorld): void { function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { const { adbCalls } = world; - assertCommandCall(adbCalls, ['exec-out', 'uiautomator', 'dump', '/dev/tty']); + assert.ok( + adbCalls.some( + (call) => + arrayEqual(call, ['exec-out', 'uiautomator', 'dump', '/dev/tty']) || + (call[0] === 'shell' && + call[1] === 'am' && + call[2] === 'instrument' && + call.includes('com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation')), + ), + JSON.stringify(adbCalls), + ); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, [ 'shell', diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index f5ecb7b2a..df39442ea 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -59,8 +59,8 @@ export async function createAndroidSettingsWorld(options?: { const inventoryRequests: DeviceInventoryRequest[] = []; const apkInstallCalls: Array<{ apkPath: string; replace?: boolean }> = []; const bundleInstallCalls: Array<{ bundlePath: string; mode: string }> = []; - let searchText = ''; - let clipboardText = 'hello'; + const shellState = createAndroidProviderShellState(); + const appState = createAndroidProviderAppState(); const spawnedLogcat: AndroidAdbProcess[] = []; const tempRoot = fs.mkdtempSync( path.join(os.tmpdir(), 'agent-device-provider-scenario-android-deploy-'), @@ -80,15 +80,14 @@ export async function createAndroidSettingsWorld(options?: { exec: async (args) => { adbCalls.push([...args]); options?.onAdbExec?.([...args]); - if (args[0] === 'shell' && args[1] === 'input' && args[2] === 'text') { - searchText = String(args[3] ?? '').replaceAll('%s', ' '); - } - if (args.join(' ') === 'shell cmd clipboard set text android otp') { - clipboardText = 'android otp'; - } - return androidAdbResult(args, searchText, clipboardText, { + updateAndroidProviderShellState(args, shellState); + const stateResult = updateAndroidProviderAppState(args, appState); + if (stateResult) return stateResult; + return androidAdbResult(args, shellState.searchText, shellState.clipboardText, { snapshotXml: options?.snapshotXml, - dumpsysWindow: options?.dumpsysWindow, + dumpsysWindow: + options?.dumpsysWindow ?? (() => androidForegroundWindowDump(appState.foreground)), + pidof: (packageName) => androidPidofResult(appState, packageName), }); }, install: async (apk, options) => { @@ -102,8 +101,8 @@ export async function createAndroidSettingsWorld(options?: { const child = makeMockAdbProcess(); spawnedLogcat.push(child); queueMicrotask(() => { - child.stdout?.push(`I/AgentDevice(4242): ${args.join(' ')}\n`); if (args.includes('logcat')) { + child.stdout?.push(`I/AgentDevice(4242): ${args.join(' ')}\n`); child.stdout?.push( [ '04-01 10:00:15.000 D/Network(4242):', @@ -118,7 +117,13 @@ export async function createAndroidSettingsWorld(options?: { '\n', ].join(' '), ); + return; } + child.stdout?.push(`I/AgentDevice(4242): ${args.join(' ')}\n`); + child.stdout?.push(null); + child.stderr?.push(null); + child.emit('exit', 0, null); + child.emit('close', 0, null); }); return child; }, @@ -126,7 +131,7 @@ export async function createAndroidSettingsWorld(options?: { if (options?.nativeTextInjection) { adbProvider.text = async (request) => { textInjectionCalls.push({ ...request }); - searchText = request.text; + shellState.searchText = request.text; }; } if (options?.nativeTouchInjection) { @@ -199,11 +204,12 @@ function androidAdbResult( options: { snapshotXml?: () => string; dumpsysWindow?: () => string; + pidof?: (packageName: string) => AndroidAdbResult | undefined; }, ): { stdout: string; stderr: string; exitCode: number; stdoutBuffer?: Buffer } { const key = args.join(' '); return ( - androidDeviceStateAdbResult(key, clipboardText) ?? + androidDeviceStateAdbResult(key, args, clipboardText, options.pidof) ?? androidMetricsAdbResult(key) ?? androidPackageAdbResult(key, args, options.dumpsysWindow) ?? androidCaptureAdbResult(key, searchText, options.snapshotXml) ?? { @@ -221,9 +227,30 @@ type AndroidAdbResult = { stdoutBuffer?: Buffer; }; +type AndroidProviderShellState = { + searchText: string; + clipboardText: string; +}; + +function createAndroidProviderShellState(): AndroidProviderShellState { + return { searchText: '', clipboardText: 'hello' }; +} + +function updateAndroidProviderShellState(args: string[], state: AndroidProviderShellState): void { + if (args[0] === 'shell' && args[1] === 'input' && args[2] === 'text') { + state.searchText = String(args[3] ?? '').replaceAll('%s', ' '); + return; + } + if (args.join(' ') === 'shell cmd clipboard set text android otp') { + state.clipboardText = 'android otp'; + } +} + function androidDeviceStateAdbResult( key: string, + args: string[], clipboardText: string, + pidof?: (packageName: string) => AndroidAdbResult | undefined, ): AndroidAdbResult | undefined { if (key === 'shell getprop sys.boot_completed') { return { stdout: '1\n', stderr: '', exitCode: 0 }; @@ -234,12 +261,80 @@ function androidDeviceStateAdbResult( if (key === 'shell dumpsys input_method') { return { stdout: 'mInputShown=false inputType=0x1\n', stderr: '', exitCode: 0 }; } - if (key === 'shell pidof com.example.demo') { - return { stdout: '4242\n', stderr: '', exitCode: 0 }; + if (args[0] === 'shell' && args[1] === 'pidof' && args[2]) { + return pidof?.(args[2]) ?? { stdout: '4242\n', stderr: '', exitCode: 0 }; } return undefined; } +type AndroidProviderAppState = { + foreground: string | null; + runningPackages: Set; +}; + +function createAndroidProviderAppState(): AndroidProviderAppState { + return { + foreground: 'com.android.settings/.Settings', + runningPackages: new Set(['com.android.settings', 'com.example.demo']), + }; +} + +function updateAndroidProviderAppState( + args: string[], + state: AndroidProviderAppState, +): AndroidAdbResult | undefined { + if (args[0] !== 'shell' || args[1] !== 'am') return undefined; + return stopAndroidProviderApp(args, state) ?? startAndroidProviderApp(args, state); +} + +function stopAndroidProviderApp( + args: string[], + state: AndroidProviderAppState, +): AndroidAdbResult | undefined { + if (args[2] !== 'force-stop' || !args[3]) return undefined; + const packageName = args[3]; + state.runningPackages.delete(packageName); + if (state.foreground?.startsWith(`${packageName}/`)) { + state.foreground = null; + } + return { stdout: '', stderr: '', exitCode: 0 }; +} + +function startAndroidProviderApp(args: string[], state: AndroidProviderAppState): undefined { + if (args[2] !== 'start' && args[2] !== 'start-activity') return undefined; + + const componentIndex = args.indexOf('-n'); + const component = componentIndex >= 0 ? args[componentIndex + 1] : undefined; + if (component) { + foregroundAndroidComponent(state, component); + return undefined; + } + if (args.includes('android.settings.SETTINGS')) { + foregroundAndroidComponent(state, 'com.android.settings/.Settings'); + } + return undefined; +} + +function foregroundAndroidComponent(state: AndroidProviderAppState, component: string): void { + const packageName = component.split('/')[0]; + if (!packageName) return; + state.runningPackages.add(packageName); + state.foreground = component; +} + +function androidPidofResult( + state: AndroidProviderAppState, + packageName: string, +): AndroidAdbResult | undefined { + return state.runningPackages.has(packageName) + ? { stdout: '4242\n', stderr: '', exitCode: 0 } + : { stdout: '', stderr: '', exitCode: 1 }; +} + +function androidForegroundWindowDump(foreground: string | null): string { + return foreground ? `mCurrentFocus=Window{42 u0 ${foreground}}\n` : 'mCurrentFocus=null\n'; +} + function androidMetricsAdbResult(key: string): AndroidAdbResult | undefined { if (key === 'shell dumpsys cpuinfo') { return { @@ -428,7 +523,7 @@ function escapeXml(value: string): string { .replaceAll('>', '>'); } -function makeMockAdbProcess(): AndroidAdbProcess { +function makeMockAdbProcess(): EventEmitter & AndroidAdbProcess { const child = new EventEmitter() as EventEmitter & AndroidAdbProcess; child.stdin = null; child.stdout = new PassThrough(); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 6f5467571..0ffadfa2a 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -483,6 +483,8 @@ agent-device settings touchid enroll agent-device settings touchid unenroll agent-device settings fingerprint match agent-device settings fingerprint nonmatch +agent-device settings clear-app-state +agent-device settings clear-app-state com.example.app agent-device settings permission grant camera agent-device settings permission deny microphone agent-device settings permission grant photos limited @@ -497,6 +499,7 @@ agent-device settings permission reset screen-recording --platform macos - Android `settings animations off|on` toggles the global `window_animation_scale`, `transition_animation_scale`, and `animator_duration_scale` values. Use it as an opt-in stabilizer for automation runs with heavy system or app animations, then restore with `settings animations on` when needed. - `settings appearance` maps to macOS appearance, iOS simulator appearance, and Android night mode. - `settings location set ` sets precise coordinates on iOS simulators and Android emulators. +- `settings clear-app-state [app-id]` clears the active session app data, or the provided app id. Android uses `pm clear`, which removes SharedPreferences, databases, files, and cache. iOS simulator removes the app data container contents. iOS physical devices and macOS are unsupported. - Face ID and Touch ID controls are iOS simulator-only. - Fingerprint simulation is supported on Android targets where `cmd fingerprint` or `adb emu finger` is available. On physical Android devices, only `cmd fingerprint` is attempted. diff --git a/website/docs/docs/quick-start.md b/website/docs/docs/quick-start.md index 1c217ee8b..341bc4e1b 100644 --- a/website/docs/docs/quick-start.md +++ b/website/docs/docs/quick-start.md @@ -142,6 +142,7 @@ Toggle device settings directly: agent-device settings wifi on agent-device settings airplane on agent-device settings appearance toggle +agent-device settings clear-app-state agent-device settings location off agent-device settings location set 37.3349 -122.009 agent-device settings permission grant camera diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index de74bacb2..b5e61e2df 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -62,7 +62,7 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, `runFlow` file/inline with `when.platform`, `when.visible`, `when.notVisible`, and limited `when.true` boolean/platform expressions, `onFlowStart` and `onFlowComplete` hooks, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn` and `longPressOn`, `inputText`, focused-field `eraseText`, and `pasteText`, `openLink`, visibility assertions and `extendedWaitUntil`, `scroll` and `scrollUntilVisible`, absolute/percentage `swipe` and `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`, and ordered trusted `runScript` file/env scripts with `http.post`, `json`, and `output` variables. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. +Currently supported areas include app launch with Apple-platform launch arguments and Android/iOS simulator `clearState`, `runFlow` file/inline with `when.platform`, `when.visible`, `when.notVisible`, and limited `when.true` boolean/platform expressions, `onFlowStart` and `onFlowComplete` hooks, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn` and `longPressOn`, `inputText`, focused-field `eraseText`, and `pasteText`, `openLink`, visibility assertions and `extendedWaitUntil`, `scroll` and `scrollUntilVisible`, absolute/percentage `swipe` and `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`, and ordered trusted `runScript` file/env scripts with `http.post`, `json`, and `output` variables. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. @@ -81,7 +81,8 @@ agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts - `context platform=...` inside each `.ad` file is the target source of truth for suite execution. - `--platform` is a filter for suite discovery; files without platform metadata are skipped when a filter is present. - `context timeout=...` and `context retries=...` can be declared per script; CLI flags override metadata. Retries are capped at `3`, and duplicate keys in the context header fail fast instead of silently overriding each other. -- By default, suite artifacts are written under `.agent-device/test-artifacts//...`. Each attempt writes `replay.ad` and `result.txt`; failed attempts also keep copied logs and artifact files when the replay produced them. +- By default, suite artifacts are written under `.agent-device/test-artifacts//...`. Each attempt writes `replay.ad`, `result.txt`, and `replay-timing.ndjson`. Failed attempts also keep copied logs and artifact files when the replay produced them. +- `replay-timing.ndjson` records attempt, cleanup, and per-step start/stop events with durations. Upload it from CI even for passing runs when comparing local and CI performance. - Timeouts are cooperative: the runner marks the attempt failed at the timeout boundary, then gives the underlying replay a short grace period to stop before session cleanup. - The default text reporter streams one-line `pass`, `fail`, or `skip` progress on stderr as each suite entry finishes or retries. Each line includes current/total suite position and elapsed seconds such as `pass 3/6 ... duration=12.34s`, then the final summary prints failed tests and passed-on-retry flaky tests; use `--verbose` to print every final result. - When `--fail-fast` and retries are both set, the current test still consumes its retries before the suite stops.