Skip to content

fix: filter covered Android snapshot surfaces #675

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thymikee merged 2 commits into from
Jun 2, 2026
Merged

fix: filter covered Android snapshot surfaces #675

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
710a4ca
fix: filter covered android snapshot surfaces
thymikee Jun 2, 2026
ecf2200
fix: guard android drawing order metadata
thymikee Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions android-snapshot-helper/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ can run.
The APK emits instrumentation status records using
`agentDeviceProtocol=android-snapshot-helper-v1`.

The XML node attributes intentionally mirror fields consumed by the host parser, including
`visible-to-user`, `drawing-order`, bounds, text/description/id, interaction booleans, and window
metadata on window roots. `drawing-order` lets the host suppress covered same-window surfaces that
the helper traversal can receive even when they are not user-reachable.

Each XML chunk is sent with:

- `outputFormat=uiautomator-xml`
Expand Down
1 change: 1 addition & 0 deletions ...elper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,7 @@ private static void appendNode(
appendNonEmptyAttribute(xml, "package", node.getPackageName());
appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription());
appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser()));
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot Jun 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Guard getDrawingOrder on API 23

On Android 6/API 23 devices this helper can still be installed because the manifest/build scripts set minSdkVersion/MIN_SDK to 23, but AccessibilityNodeInfo.getDrawingOrder() is only available starting API 24 per the Android reference. In that supported environment, serializing the first node will throw NoSuchMethodError and the snapshot instrumentation fails instead of returning XML; gate this call on Build.VERSION.SDK_INT >= 24 or omit the attribute on API 23.

Useful? React with 👍 / 👎.

appendTrueAttribute(xml, "clickable", node.isClickable());
appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled()));
appendTrueAttribute(xml, "focusable", node.isFocusable());
Expand Down
123 changes: 119 additions & 4 deletions src/platforms/android/__tests__/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,17 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => {
assert.equal(result.nodes[0]!.label, 'Line 1\nLine 2\t&<>"\'');
});

test('parseUiHierarchy reads Android bounds with negative coordinates', () => {
const xml =
'<hierarchy><node class="android.widget.TextView" text="Clipped" bounds="[0,935][-67,994]"/></hierarchy>';

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.deepEqual(result.nodes[0]!.rect, { x: 0, y: 935, width: 0, height: 59 });
});

test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
const xml =
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" drawing-order="4" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';

assert.deepEqual(Array.from(androidUiNodes(xml)), [
{
Expand All @@ -177,6 +185,7 @@ test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
clickable: false,
enabled: true,
visibleToUser: true,
drawingOrder: 4,
focusable: true,
focused: true,
password: true,
Expand Down Expand Up @@ -272,7 +281,7 @@ test('parseUiHierarchy excludes Android nodes that are not visible to the user',
);
});

test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapshots', () => {
test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="false"/>
Expand All @@ -281,8 +290,114 @@ test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapsho

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(result.nodes[0]!.visibleToUser, true);
assert.equal(result.nodes[1]!.label, 'Hidden drawer action');
assert.equal(result.nodes[1]!.visibleToUser, false);
assert.equal(
result.nodes.some((node) => node.label === 'Hidden drawer action'),
false,
);
});

test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" enabled="true" visible-to-user="false">
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="true"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Hidden drawer action'),
false,
);
});

test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.ScrollView" bounds="[0,120][300,844]" scrollable="true" clickable="true" enabled="true" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Hidden drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Foreground action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Hidden drawer action'),
false,
);
});

test('parseUiHierarchy keeps visible side-by-side drawer and content subtrees', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][120,844]" visible-to-user="true" drawing-order="2">
<node class="android.widget.Button" text="Visible drawer action" bounds="[0,220][110,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
<node class="android.view.ViewGroup" bounds="[120,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Visible content action" bounds="[150,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Visible drawer action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Visible content action'),
true,
);
});

test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unavailable', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
<node class="android.widget.Button" text="Legacy drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Foreground action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Legacy drawer action'),
true,
);
});

test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible overlays', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2"/>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Still visible action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Still visible action'),
true,
);
});

test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
Expand Down
121 changes: 120 additions & 1 deletion src/platforms/android/ui-hierarchy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export type AndroidUiNodeMetadata = {
clickable?: boolean;
enabled?: boolean;
visibleToUser?: boolean;
drawingOrder?: number;
focusable?: boolean;
focused?: boolean;
password?: boolean;
Expand Down Expand Up @@ -244,6 +245,7 @@ function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
focused: boolAttr('focused'),
password: boolAttr('password'),
...optionalBoolAttr('visibleToUser', 'visible-to-user'),
...optionalNumberAttr('drawingOrder', 'drawing-order'),
...optionalBoolAttr('scrollable', 'scrollable'),
...optionalBoolAttr('canScrollForward', 'can-scroll-forward'),
...optionalBoolAttr('canScrollBackward', 'can-scroll-backward'),
Expand Down Expand Up @@ -383,7 +385,7 @@ function readXmlAttr(attrs: Map<string, string>, name: string): string | null {

function parseBounds(bounds: string | null): Rect | undefined {
if (!bounds) return undefined;
const match = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(bounds);
const match = /\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/.exec(bounds);
if (!match) return undefined;
const x1 = Number(match[1]);
const y1 = Number(match[2]);
Expand All @@ -401,6 +403,7 @@ export type AndroidUiHierarchy = {
rect?: Rect;
enabled?: boolean;
visibleToUser?: boolean;
drawingOrder?: number;
hittable?: boolean;
depth: number;
parentIndex?: number;
Expand Down Expand Up @@ -428,6 +431,15 @@ type AndroidNodeInclusionInfo = {
isVisual: boolean;
};

type AndroidTreePruneState = {
agentVisibleContentMemo: WeakMap<AndroidNode, boolean>;
};

type AndroidCoveringCandidate = AndroidNode & {
rect: Rect;
drawingOrder: number;
};

const ANDROID_WINDOW_TYPE_APPLICATION = 1;

export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
Expand Down Expand Up @@ -461,6 +473,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
rect: attrs.rect,
enabled: attrs.enabled,
visibleToUser: attrs.visibleToUser,
drawingOrder: attrs.drawingOrder,
hittable: attrs.clickable ?? attrs.focusable,
scrollable: attrs.scrollable,
canScrollForward: attrs.canScrollForward,
Expand All @@ -481,11 +494,117 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
}
match = tokenRegex.exec(xml);
}
// Raw Android snapshots are uncollapsed, but still agent-visible. The helper can expose
// aria-hidden/no-hide-descendants children, so prune nodes Android marks hidden to users.
pruneAndroidInvisibleSubtrees(root);
discardInactiveAndroidApplicationWindows(root);
// UiAutomation can expose covered React Native navigation surfaces in the same accessibility
// window. If a higher drawing-order sibling covers them, agents should see the foreground surface.
pruneAndroidCoveredSubtrees(root, { agentVisibleContentMemo: new WeakMap() });
applyAndroidScrollActionHints(root);
return root;
}

function pruneAndroidInvisibleSubtrees(node: AndroidNode): void {
let keptCount = 0;
for (const child of node.children) {
if (child.visibleToUser === false) continue;
pruneAndroidInvisibleSubtrees(child);
node.children[keptCount] = child;
keptCount += 1;
}
if (keptCount < node.children.length) {
node.children.length = keptCount;
}
}

function pruneAndroidCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneState): void {
for (const child of node.children) {
pruneAndroidCoveredSubtrees(child, state);
}
if (node.children.length < 2) {
return;
}
const siblings = node.children;
const coveringCandidates = siblings.filter((sibling) => canCoverSibling(sibling, state));
if (coveringCandidates.length === 0) return;
node.children = siblings.filter(
(child) => !isCoveredByHigherDrawingOrderSibling(child, coveringCandidates),
);
}

function isCoveredByHigherDrawingOrderSibling(
node: AndroidNode,
coveringCandidates: AndroidCoveringCandidate[],
): boolean {
if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) {
return false;
}

for (const sibling of coveringCandidates) {
if (sibling === node || sibling.drawingOrder <= node.drawingOrder) {
continue;
}
if (rectCoverage(sibling.rect, node.rect) >= 0.9) {
return true;
}
}
return false;
}

function canCoverSibling(
node: AndroidNode,
state: AndroidTreePruneState,
): node is AndroidCoveringCandidate {
return (
node.visibleToUser !== false &&
node.drawingOrder !== undefined &&
hasPositiveRect(node) &&
hasAgentVisibleContent(node, state)
);
}

function hasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
const cached = state.agentVisibleContentMemo.get(node);
if (cached !== undefined) return cached;

const result = computeHasAgentVisibleContent(node, state);
state.agentVisibleContentMemo.set(node, result);
return result;
}

function computeHasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
if (node.visibleToUser === false) return false;
if (node.hittable) return true;
const label = node.label?.trim() ?? '';
if (label && !isGenericAndroidId(label)) return true;
const identifier = node.identifier?.trim() ?? '';
if (identifier && !isGenericAndroidId(identifier)) return true;
return node.children.some((child) => hasAgentVisibleContent(child, state));
}

function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } {
return Boolean(node.rect && node.rect.width > 0 && node.rect.height > 0);
}

function rectCoverage(coveringRect: Rect, targetRect: Rect): number {
const targetArea = targetRect.width * targetRect.height;
if (targetArea <= 0) return 0;
return intersectionArea(coveringRect, targetRect) / targetArea;
}

function intersectionArea(left: Rect, right: Rect): number {
const xOverlap = Math.max(
0,
Math.min(left.x + left.width, right.x + right.width) - Math.max(left.x, right.x),
);
const yOverlap = Math.max(
0,
Math.min(left.y + left.height, right.y + right.height) - Math.max(left.y, right.y),
);
return xOverlap * yOverlap;
}

function applyAndroidScrollActionHints(root: AndroidUiHierarchy): void {
const stack = [...root.children];
while (stack.length > 0) {
Expand Down
Loading