Skip to content

fix: filter covered Android snapshot surfaces #675

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thymikee merged 2 commits into from
Jun 2, 2026
Merged

fix: filter covered Android snapshot surfaces #675

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
710a4ca
fix: filter covered android snapshot surfaces
thymikee Jun 2, 2026
ecf2200
fix: guard android drawing order metadata
thymikee Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions android-snapshot-helper/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ can run.
The APK emits instrumentation status records using
`agentDeviceProtocol=android-snapshot-helper-v1`.

The XML node attributes intentionally mirror fields consumed by the host parser, including
`visible-to-user`, `drawing-order`, bounds, text/description/id, interaction booleans, and window
metadata on window roots. `drawing-order` lets the host suppress covered same-window surfaces that
the helper traversal can receive even when they are not user-reachable. The helper emits
`drawing-order` on Android API 24+ and omits it on API 23, where the platform API is unavailable.

Each XML chunk is sent with:

- `outputFormat=uiautomator-xml`
Expand Down
8 changes: 8 additions & 0 deletions ...elper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import android.app.Instrumentation;
import android.app.UiAutomation;
import android.graphics.Rect;
import android.os.Build;
import android.os.Bundle;
import android.util.Base64;
import android.view.accessibility.AccessibilityNodeInfo;
Expand Down Expand Up @@ -513,6 +514,7 @@ private static void appendNode(
appendNonEmptyAttribute(xml, "package", node.getPackageName());
appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription());
appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser()));
appendDrawingOrderAttribute(xml, node);
appendTrueAttribute(xml, "clickable", node.isClickable());
appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled()));
appendTrueAttribute(xml, "focusable", node.isFocusable());
Expand Down Expand Up @@ -584,6 +586,12 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean
}
}

private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
}
}

private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metadata) {
appendAttribute(xml, "window-index", Integer.toString(metadata.index));
appendAttribute(xml, "window-type", Integer.toString(metadata.type));
Expand Down
169 changes: 165 additions & 4 deletions src/platforms/android/__tests__/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,17 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => {
assert.equal(result.nodes[0]!.label, 'Line 1\nLine 2\t&<>"\'');
});

test('parseUiHierarchy reads Android bounds with negative coordinates', () => {
const xml =
'<hierarchy><node class="android.widget.TextView" text="Clipped" bounds="[0,935][-67,994]"/></hierarchy>';

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.deepEqual(result.nodes[0]!.rect, { x: 0, y: 935, width: 0, height: 59 });
});

test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
const xml =
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" drawing-order="4" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';

assert.deepEqual(Array.from(androidUiNodes(xml)), [
{
Expand All @@ -177,6 +185,7 @@ test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
clickable: false,
enabled: true,
visibleToUser: true,
drawingOrder: 4,
focusable: true,
focused: true,
password: true,
Expand Down Expand Up @@ -272,7 +281,7 @@ test('parseUiHierarchy excludes Android nodes that are not visible to the user',
);
});

test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapshots', () => {
test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="false"/>
Expand All @@ -281,8 +290,160 @@ test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapsho

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(result.nodes[0]!.visibleToUser, true);
assert.equal(result.nodes[1]!.label, 'Hidden drawer action');
assert.equal(result.nodes[1]!.visibleToUser, false);
assert.equal(
result.nodes.some((node) => node.label === 'Hidden drawer action'),
false,
);
});

test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" enabled="true" visible-to-user="false">
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="true"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Hidden drawer action'),
false,
);
});

test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.ScrollView" bounds="[0,120][300,844]" scrollable="true" clickable="true" enabled="true" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Hidden drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Foreground action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Hidden drawer action'),
false,
);
});

test('parseUiHierarchy keeps visible side-by-side drawer and content subtrees', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][120,844]" visible-to-user="true" drawing-order="2">
<node class="android.widget.Button" text="Visible drawer action" bounds="[0,220][110,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
<node class="android.view.ViewGroup" bounds="[120,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Visible content action" bounds="[150,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Visible drawer action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Visible content action'),
true,
);
});

test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unavailable', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
<node class="android.widget.Button" text="Legacy drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Foreground action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Legacy drawer action'),
true,
);
});

test('parseUiHierarchy keeps overlapping siblings when drawing-order ties', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="First tied action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Second tied action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'First tied action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Second tied action'),
true,
);
});

test('parseUiHierarchy keeps lower siblings below the covered-area threshold', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,717]" visible-to-user="true" drawing-order="2">
<node class="android.widget.Button" text="Partial overlay action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Mostly visible action" bounds="[0,760][280,820]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Partial overlay action'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Mostly visible action'),
true,
);
});

test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible overlays', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2"/>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.Button" text="Still visible action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Still visible action'),
true,
);
});

test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
Expand Down
Loading
Loading