Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ public final class SnapshotInstrumentation extends Instrumentation {
private static final String OUTPUT_FORMAT = "uiautomator-xml";
private static final String HELPER_API_VERSION = "1";
private static final int CHUNK_SIZE = 2 * 1024;
// Keep the default quiet window short: RN/animation-heavy apps often never become fully idle,
// and callers can still override this for alert-style flows that need a longer settle period.
private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 25;
private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 25;
// Match the host defaults: long enough to avoid mid-transition RN snapshots, but still bounded
// below the stock uiautomator idle wait so busy apps do not stall every capture.
private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 500;
private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 100;
private static final long DEFAULT_TIMEOUT_MS = 8_000;
private static final int DEFAULT_MAX_DEPTH = 128;
private static final int DEFAULT_MAX_NODES = 5_000;
Expand Down Expand Up @@ -351,7 +351,7 @@ private CaptureResult captureXml(
AccessibilityNodeInfo root = automation.getRootInActiveWindow();
try {
if (root != null) {
appendNode(xml, root, 0, 0, maxDepth, maxNodes, stats);
appendNode(xml, root, 0, 0, maxDepth, maxNodes, stats, null);
windowCount = 1;
}
captureMode = "active-window";
Expand Down Expand Up @@ -439,7 +439,15 @@ private static int appendInteractiveWindowRoots(
}
StringBuilder windowXml = new StringBuilder();
CaptureStats windowStats = stats.copy();
appendNode(windowXml, root, windowCount, 0, maxDepth, maxNodes, windowStats);
appendNode(
windowXml,
root,
windowCount,
0,
maxDepth,
maxNodes,
windowStats,
readWindowMetadata(window, windowCount));
xml.append(windowXml);
stats.copyFrom(windowStats);
windowCount += 1;
Expand Down Expand Up @@ -482,7 +490,8 @@ private static void appendNode(
int depth,
int maxDepth,
int maxNodes,
CaptureStats stats) {
CaptureStats stats,
WindowMetadata windowMetadata) {
if (stats.nodeCount >= maxNodes) {
stats.truncated = true;
return;
Expand All @@ -495,11 +504,15 @@ private static void appendNode(
// without affecting current snapshot semantics; add fields back here when TS starts reading
// them.
appendAttribute(xml, "index", Integer.toString(nodeIndex));
if (windowMetadata != null) {
appendWindowMetadata(xml, windowMetadata);
}
appendNonEmptyAttribute(xml, "text", node.getText());
appendNonEmptyAttribute(xml, "resource-id", node.getViewIdResourceName());
appendAttribute(xml, "class", node.getClassName());
appendNonEmptyAttribute(xml, "package", node.getPackageName());
appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription());
appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser()));
appendTrueAttribute(xml, "clickable", node.isClickable());
appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled()));
appendTrueAttribute(xml, "focusable", node.isFocusable());
Expand Down Expand Up @@ -550,7 +563,7 @@ private static void appendNode(
continue;
}
try {
appendNode(xml, child, index, depth + 1, maxDepth, maxNodes, stats);
appendNode(xml, child, index, depth + 1, maxDepth, maxNodes, stats, null);
} finally {
child.recycle();
}
Expand All @@ -571,6 +584,32 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean
}
}

private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metadata) {
appendAttribute(xml, "window-index", Integer.toString(metadata.index));
appendAttribute(xml, "window-type", Integer.toString(metadata.type));
appendAttribute(xml, "window-layer", Integer.toString(metadata.layer));
appendAttribute(xml, "window-active", Boolean.toString(metadata.active));
appendAttribute(xml, "window-focused", Boolean.toString(metadata.focused));
appendAttribute(
xml,
"window-bounds",
String.format(
Locale.ROOT,
"[%d,%d][%d,%d]",
metadata.bounds.left,
metadata.bounds.top,
metadata.bounds.right,
metadata.bounds.bottom));
}

@SuppressWarnings("deprecation")
private static WindowMetadata readWindowMetadata(AccessibilityWindowInfo window, int index) {
Rect bounds = new Rect();
window.getBoundsInScreen(bounds);
return new WindowMetadata(
index, window.getType(), window.getLayer(), window.isActive(), window.isFocused(), bounds);
}

private static void appendAttribute(StringBuilder xml, String name, CharSequence value) {
String stringValue = value == null ? "" : value.toString();
xml.append(' ');
Expand Down Expand Up @@ -702,4 +741,22 @@ private static final class CaptureResult {
this.truncated = truncated;
}
}

private static final class WindowMetadata {
final int index;
final int type;
final int layer;
final boolean active;
final boolean focused;
final Rect bounds;

WindowMetadata(int index, int type, int layer, boolean active, boolean focused, Rect bounds) {
this.index = index;
this.type = type;
this.layer = layer;
this.active = active;
this.focused = focused;
this.bounds = bounds;
}
}
}
9 changes: 9 additions & 0 deletions src/__tests__/cli-grammar.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,13 @@ test('settings grammar owns positional parsing for CLI commands', () => {
assert.equal(location.state, 'set');
assert.equal(location.latitude, 37.3349);
assert.equal(location.longitude, -122.009);

const clearAppState = readInputFromCli('settings', ['clear-app-state', 'com.example.app'], {
...BASE_FLAGS,
platform: 'android',
});
assert.equal(clearAppState.platform, 'android');
assert.equal(clearAppState.setting, 'clear-app-state');
assert.equal(clearAppState.state, 'clear');
assert.equal(clearAppState.app, 'com.example.app');
});
30 changes: 30 additions & 0 deletions src/__tests__/runtime-interactions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,36 @@ test('runtime directional swipe uses the visible viewport instead of off-screen
assert.deepEqual(calls, [{ from: { x: 50, y: 50 }, to: { x: 25, y: 50 } }]);
});

test('runtime gesture swipe presets use stable viewport lanes', async () => {
const calls: unknown[] = [];
const device = createInteractionDevice(snapshotWithOffscreenContent(), {
platform: 'android',
swipe: async (_context, from, to, options) => {
calls.push({ from, to, durationMs: options?.durationMs });
},
});

const pageSwipe = await device.interactions.swipe({
preset: 'left',
durationMs: 300,
session: 'default',
});
const edgeSwipe = await device.interactions.swipe({
preset: 'right-edge',
durationMs: 350,
session: 'default',
});

assert.deepEqual(pageSwipe.from, { x: 90, y: 65 });
assert.deepEqual(pageSwipe.to, { x: 10, y: 65 });
assert.deepEqual(edgeSwipe.from, { x: 8, y: 50 });
assert.deepEqual(edgeSwipe.to, { x: 85, y: 50 });
assert.deepEqual(calls, [
{ from: { x: 90, y: 65 }, to: { x: 10, y: 65 }, durationMs: 300 },
{ from: { x: 8, y: 50 }, to: { x: 85, y: 50 }, durationMs: 350 },
]);
});

test('runtime viewport gestures reject inspect-only macOS surfaces', async () => {
for (const surface of ['desktop', 'menubar'] as const) {
const device = createInteractionDevice(selectorSnapshot(), {
Expand Down
11 changes: 11 additions & 0 deletions src/client-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,11 @@ export type FlingOptions = ClientCommandBaseOptions & {
durationMs?: number;
};

export type SwipeGestureOptions = ClientCommandBaseOptions & {
preset: 'left' | 'right' | 'left-edge' | 'right-edge';
durationMs?: number;
};

export type FocusOptions = ClientCommandBaseOptions & {
x: number;
y: number;
Expand Down Expand Up @@ -763,6 +768,11 @@ export type PermissionTarget =
| 'input-monitoring';

export type SettingsUpdateOptions =
| (ClientCommandBaseOptions & {
setting: 'clear-app-state';
state: 'clear';
app?: string;
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Thread app through the structured settings input

This adds an explicit app option for settings clean-user-data, but the structured settings command still only declares/reads setting/state/latitude/longitude/permission/mode in src/commands/client-command-contracts.ts, so MCP/command-surface callers that pass { app: 'com.foo' } have it omitted by readFieldInput and the daemon receives no target unless there is an active session. In the documented scenario of clearing a specific app id from structured clients, the command fails with “requires an app id or an active app session”; add app to the settings fields/schema and pass it through.

Useful? React with 👍 / 👎.

})
| (ClientCommandBaseOptions & {
setting: 'wifi' | 'airplane' | 'location';
state: 'on' | 'off';
Expand Down Expand Up @@ -915,6 +925,7 @@ export type AgentDeviceClient = {
swipe: (options: SwipeOptions) => Promise<CommandRequestResult>;
pan: (options: PanOptions) => Promise<CommandRequestResult>;
fling: (options: FlingOptions) => Promise<CommandRequestResult>;
swipeGesture: (options: SwipeGestureOptions) => Promise<CommandRequestResult>;
focus: (options: FocusOptions) => Promise<CommandRequestResult>;
type: (options: TypeTextOptions) => Promise<CommandRequestResult>;
fill: (options: FillOptions) => Promise<CommandRequestResult>;
Expand Down
1 change: 1 addition & 0 deletions src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,7 @@ export function createAgentDeviceClient(
swipe: async (options) => await executeCommand('swipe', options),
pan: async (options) => await executeCommand('gesture-pan', options),
fling: async (options) => await executeCommand('gesture-fling', options),
swipeGesture: async (options) => await executeCommand('gesture-swipe', options),
focus: async (options) => await executeCommand('focus', options),
type: async (options) => await executeCommand('type', options),
fill: async (options) => await executeCommand('fill', options),
Expand Down
2 changes: 1 addition & 1 deletion src/command-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ const LOCAL_CLI_COMMANDS = {
session: 'session',
} as const;

const GESTURE_SUBCOMMANDS = ['pan', 'fling', 'pinch', 'rotate', 'transform'] as const;
const GESTURE_SUBCOMMANDS = ['pan', 'fling', 'swipe', 'pinch', 'rotate', 'transform'] as const;
export const GESTURE_SUBCOMMAND_ERROR = `gesture requires one of: ${GESTURE_SUBCOMMANDS.join(', ')}`;

export type PublicCommandName = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS];
Expand Down
7 changes: 7 additions & 0 deletions src/commands/cli-grammar/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -231,10 +231,17 @@ function readSettingsOptionsFromPositionals(
mode: readPermissionMode(positionals[3]),
};
}
if (setting === 'clear-app-state') {
const app = state === 'clear' ? positionals[2] : state;
return { ...base, setting, state: 'clear', app };
}
throw new AppError('INVALID_ARGS', 'Invalid settings arguments.');
}

function settingsPositionals(input: SettingsUpdateOptions): string[] {
if (input.setting === 'clear-app-state') {
return [input.setting, ...optionalString(input.app)];
}
if (input.setting === 'location' && input.state === 'set') {
return [input.setting, input.state, String(input.latitude), String(input.longitude)];
}
Expand Down
22 changes: 20 additions & 2 deletions src/commands/cli-grammar/gesture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export const gestureDaemonWriters = {
'gesture-fling': direct(PUBLIC_COMMANDS.gesture, (input) =>
flingPositionals(input as FlingOptions),
),
'gesture-swipe': direct(PUBLIC_COMMANDS.gesture, swipePresetPositionals),
'gesture-pinch': direct(PUBLIC_COMMANDS.gesture, pinchPositionals),
'gesture-rotate': direct(PUBLIC_COMMANDS.gesture, (input) =>
rotateGesturePositionals(input as RotateGestureOptions),
Expand Down Expand Up @@ -49,6 +50,8 @@ function gesturePositionals(input: CommandInput): string[] {
...optionalNumber(input.distance),
...optionalNumber(input.durationMs),
];
case 'swipe':
return swipePresetPositionals(input);
case 'pinch':
return [
'pinch',
Expand Down Expand Up @@ -78,11 +81,19 @@ function gesturePositionals(input: CommandInput): string[] {
default:
throw new AppError(
'INVALID_ARGS',
'gesture requires pan, fling, pinch, rotate, or transform',
'gesture requires pan, fling, swipe, pinch, rotate, or transform',
);
}
}

function swipePresetPositionals(input: CommandInput): string[] {
return [
'swipe',
requiredDaemonString(input.preset, 'gesture swipe requires preset'),
...optionalNumber(input.durationMs),
];
}

function panPositionals(input: CommandInput): string[] {
return [
'pan',
Expand Down Expand Up @@ -153,6 +164,13 @@ function gestureInputFromCli(positionals: string[], flags: CliFlags): Record<str
distance: optionalCliNumber(args[3]),
durationMs: optionalCliNumber(args[4]),
};
case 'swipe':
return {
...common,
kind: subcommand,
preset: args[0],
durationMs: optionalCliNumber(args[1]),
};
case 'pinch':
return {
...common,
Expand Down Expand Up @@ -187,7 +205,7 @@ function gestureInputFromCli(positionals: string[], flags: CliFlags): Record<str
default:
throw new AppError(
'INVALID_ARGS',
'gesture requires pan, fling, pinch, rotate, or transform',
'gesture requires pan, fling, swipe, pinch, rotate, or transform',
);
}
}
Expand Down
12 changes: 12 additions & 0 deletions src/commands/interaction-command-contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type {
LongPressOptions,
RotateGestureOptions,
ScrollOptions,
SwipeGestureOptions,
SwipeOptions,
TransformGestureOptions,
TypeTextOptions,
Expand All @@ -35,6 +36,7 @@ import {
type PinchInput,
type PressInput,
type RotateInput,
type SwipeGestureInput,
type TransformInput,
} from './interaction-command-metadata.ts';

Expand Down Expand Up @@ -81,6 +83,8 @@ export const interactionCommandDefinitions = [
return await client.interactions.pan(toPanOptions(input));
case 'fling':
return await client.interactions.fling(toFlingOptions(input));
case 'swipe':
return await client.interactions.swipeGesture(toSwipeGestureOptions(input));
case 'pinch':
return await client.interactions.pinch(toPinchOptions(input));
case 'rotate':
Expand Down Expand Up @@ -168,6 +172,14 @@ function toFlingOptions(input: FlingInput): FlingOptions {
};
}

function toSwipeGestureOptions(input: SwipeGestureInput): SwipeGestureOptions {
return {
...commonToClientOptions(input),
preset: input.preset,
durationMs: input.durationMs,
};
}

function toPinchOptions(input: PinchInput): PinchOptions {
return {
...commonToClientOptions(input),
Expand Down
Loading
Loading