Skip to content

Commit 6f637d0

Browse files
thymikeethymikee
authored
fix: filter covered Android snapshot surfaces (#675)
* fix: filter covered android snapshot surfaces * fix: guard android drawing order metadata
1 parent 0313262 commit 6f637d0

4 files changed

Lines changed: 299 additions & 5 deletions

File tree

‎android-snapshot-helper/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,12 @@ can run.
4848
The APK emits instrumentation status records using
4949
`agentDeviceProtocol=android-snapshot-helper-v1`.
5050

51+
The XML node attributes intentionally mirror fields consumed by the host parser, including
52+
`visible-to-user`, `drawing-order`, bounds, text/description/id, interaction booleans, and window
53+
metadata on window roots. `drawing-order` lets the host suppress covered same-window surfaces that
54+
the helper traversal can receive even when they are not user-reachable. The helper emits
55+
`drawing-order` on Android API 24+ and omits it on API 23, where the platform API is unavailable.
56+
5157
Each XML chunk is sent with:
5258

5359
- `outputFormat=uiautomator-xml`

‎android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import android.app.Instrumentation;
55
import android.app.UiAutomation;
66
import android.graphics.Rect;
7+
import android.os.Build;
78
import android.os.Bundle;
89
import android.util.Base64;
910
import android.view.accessibility.AccessibilityNodeInfo;
@@ -513,6 +514,7 @@ private static void appendNode(
513514
appendNonEmptyAttribute(xml, "package", node.getPackageName());
514515
appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription());
515516
appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser()));
517+
appendDrawingOrderAttribute(xml, node);
516518
appendTrueAttribute(xml, "clickable", node.isClickable());
517519
appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled()));
518520
appendTrueAttribute(xml, "focusable", node.isFocusable());
@@ -584,6 +586,12 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean
584586
}
585587
}
586588

589+
private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) {
590+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
591+
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
592+
}
593+
}
594+
587595
private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metadata) {
588596
appendAttribute(xml, "window-index", Integer.toString(metadata.index));
589597
appendAttribute(xml, "window-type", Integer.toString(metadata.type));

‎src/platforms/android/__tests__/index.test.ts‎

Lines changed: 165 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,17 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => {
161161
assert.equal(result.nodes[0]!.label, 'Line 1\nLine 2\t&<>"\'');
162162
});
163163

164+
test('parseUiHierarchy reads Android bounds with negative coordinates', () => {
165+
const xml =
166+
'<hierarchy><node class="android.widget.TextView" text="Clipped" bounds="[0,935][-67,994]"/></hierarchy>';
167+
168+
const result = parseUiHierarchy(xml, 800, { raw: true });
169+
assert.deepEqual(result.nodes[0]!.rect, { x: 0, y: 935, width: 0, height: 59 });
170+
});
171+
164172
test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
165173
const xml =
166-
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';
174+
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" drawing-order="4" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';
167175

168176
assert.deepEqual(Array.from(androidUiNodes(xml)), [
169177
{
@@ -177,6 +185,7 @@ test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
177185
clickable: false,
178186
enabled: true,
179187
visibleToUser: true,
188+
drawingOrder: 4,
180189
focusable: true,
181190
focused: true,
182191
password: true,
@@ -272,7 +281,7 @@ test('parseUiHierarchy excludes Android nodes that are not visible to the user',
272281
);
273282
});
274283

275-
test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapshots', () => {
284+
test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => {
276285
const xml = `<hierarchy>
277286
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
278287
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="false"/>
@@ -281,8 +290,160 @@ test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapsho
281290

282291
const result = parseUiHierarchy(xml, 800, { raw: true });
283292
assert.equal(result.nodes[0]!.visibleToUser, true);
284-
assert.equal(result.nodes[1]!.label, 'Hidden drawer action');
285-
assert.equal(result.nodes[1]!.visibleToUser, false);
293+
assert.equal(
294+
result.nodes.some((node) => node.label === 'Hidden drawer action'),
295+
false,
296+
);
297+
});
298+
299+
test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => {
300+
const xml = `<hierarchy>
301+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
302+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" enabled="true" visible-to-user="false">
303+
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="true"/>
304+
</node>
305+
</node>
306+
</hierarchy>`;
307+
308+
const result = parseUiHierarchy(xml, 800, { raw: true });
309+
assert.equal(
310+
result.nodes.some((node) => node.label === 'Hidden drawer action'),
311+
false,
312+
);
313+
});
314+
315+
test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => {
316+
const xml = `<hierarchy>
317+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
318+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
319+
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
320+
</node>
321+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
322+
<node class="android.widget.ScrollView" bounds="[0,120][300,844]" scrollable="true" clickable="true" enabled="true" visible-to-user="true" drawing-order="1">
323+
<node class="android.widget.Button" text="Hidden drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
324+
</node>
325+
</node>
326+
</node>
327+
</hierarchy>`;
328+
329+
const result = parseUiHierarchy(xml, 800, { raw: true });
330+
assert.equal(
331+
result.nodes.some((node) => node.label === 'Foreground action'),
332+
true,
333+
);
334+
assert.equal(
335+
result.nodes.some((node) => node.label === 'Hidden drawer action'),
336+
false,
337+
);
338+
});
339+
340+
test('parseUiHierarchy keeps visible side-by-side drawer and content subtrees', () => {
341+
const xml = `<hierarchy>
342+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
343+
<node class="android.view.ViewGroup" bounds="[0,0][120,844]" visible-to-user="true" drawing-order="2">
344+
<node class="android.widget.Button" text="Visible drawer action" bounds="[0,220][110,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
345+
</node>
346+
<node class="android.view.ViewGroup" bounds="[120,0][390,844]" visible-to-user="true" drawing-order="1">
347+
<node class="android.widget.Button" text="Visible content action" bounds="[150,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
348+
</node>
349+
</node>
350+
</hierarchy>`;
351+
352+
const result = parseUiHierarchy(xml, 800, { raw: true });
353+
assert.equal(
354+
result.nodes.some((node) => node.label === 'Visible drawer action'),
355+
true,
356+
);
357+
assert.equal(
358+
result.nodes.some((node) => node.label === 'Visible content action'),
359+
true,
360+
);
361+
});
362+
363+
test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unavailable', () => {
364+
const xml = `<hierarchy>
365+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true">
366+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
367+
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true"/>
368+
</node>
369+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
370+
<node class="android.widget.Button" text="Legacy drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true"/>
371+
</node>
372+
</node>
373+
</hierarchy>`;
374+
375+
const result = parseUiHierarchy(xml, 800, { raw: true });
376+
assert.equal(
377+
result.nodes.some((node) => node.label === 'Foreground action'),
378+
true,
379+
);
380+
assert.equal(
381+
result.nodes.some((node) => node.label === 'Legacy drawer action'),
382+
true,
383+
);
384+
});
385+
386+
test('parseUiHierarchy keeps overlapping siblings when drawing-order ties', () => {
387+
const xml = `<hierarchy>
388+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
389+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
390+
<node class="android.widget.Button" text="First tied action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
391+
</node>
392+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
393+
<node class="android.widget.Button" text="Second tied action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
394+
</node>
395+
</node>
396+
</hierarchy>`;
397+
398+
const result = parseUiHierarchy(xml, 800, { raw: true });
399+
assert.equal(
400+
result.nodes.some((node) => node.label === 'First tied action'),
401+
true,
402+
);
403+
assert.equal(
404+
result.nodes.some((node) => node.label === 'Second tied action'),
405+
true,
406+
);
407+
});
408+
409+
test('parseUiHierarchy keeps lower siblings below the covered-area threshold', () => {
410+
const xml = `<hierarchy>
411+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
412+
<node class="android.view.ViewGroup" bounds="[0,0][390,717]" visible-to-user="true" drawing-order="2">
413+
<node class="android.widget.Button" text="Partial overlay action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
414+
</node>
415+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
416+
<node class="android.widget.Button" text="Mostly visible action" bounds="[0,760][280,820]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
417+
</node>
418+
</node>
419+
</hierarchy>`;
420+
421+
const result = parseUiHierarchy(xml, 800, { raw: true });
422+
assert.equal(
423+
result.nodes.some((node) => node.label === 'Partial overlay action'),
424+
true,
425+
);
426+
assert.equal(
427+
result.nodes.some((node) => node.label === 'Mostly visible action'),
428+
true,
429+
);
430+
});
431+
432+
test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible overlays', () => {
433+
const xml = `<hierarchy>
434+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
435+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2"/>
436+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
437+
<node class="android.widget.Button" text="Still visible action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
438+
</node>
439+
</node>
440+
</hierarchy>`;
441+
442+
const result = parseUiHierarchy(xml, 800, { raw: true });
443+
assert.equal(
444+
result.nodes.some((node) => node.label === 'Still visible action'),
445+
true,
446+
);
286447
});
287448

288449
test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {

0 commit comments

Comments
 (0)