Skip to content

Commit 710a4ca

Browse files
thymikeethymikee
committed
fix: filter covered android snapshot surfaces
1 parent 0313262 commit 710a4ca

4 files changed

Lines changed: 245 additions & 5 deletions

File tree

‎android-snapshot-helper/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ can run.
4848
The APK emits instrumentation status records using
4949
`agentDeviceProtocol=android-snapshot-helper-v1`.
5050

51+
The XML node attributes intentionally mirror fields consumed by the host parser, including
52+
`visible-to-user`, `drawing-order`, bounds, text/description/id, interaction booleans, and window
53+
metadata on window roots. `drawing-order` lets the host suppress covered same-window surfaces that
54+
the helper traversal can receive even when they are not user-reachable.
55+
5156
Each XML chunk is sent with:
5257

5358
- `outputFormat=uiautomator-xml`

‎android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,7 @@ private static void appendNode(
513513
appendNonEmptyAttribute(xml, "package", node.getPackageName());
514514
appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription());
515515
appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser()));
516+
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
516517
appendTrueAttribute(xml, "clickable", node.isClickable());
517518
appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled()));
518519
appendTrueAttribute(xml, "focusable", node.isFocusable());

‎src/platforms/android/__tests__/index.test.ts‎

Lines changed: 119 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,17 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => {
161161
assert.equal(result.nodes[0]!.label, 'Line 1\nLine 2\t&<>"\'');
162162
});
163163

164+
test('parseUiHierarchy reads Android bounds with negative coordinates', () => {
165+
const xml =
166+
'<hierarchy><node class="android.widget.TextView" text="Clipped" bounds="[0,935][-67,994]"/></hierarchy>';
167+
168+
const result = parseUiHierarchy(xml, 800, { raw: true });
169+
assert.deepEqual(result.nodes[0]!.rect, { x: 0, y: 935, width: 0, height: 59 });
170+
});
171+
164172
test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
165173
const xml =
166-
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';
174+
'<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" visible-to-user="true" drawing-order="4" focusable="true" focused="true" password="true" window-index="0" window-type="1" window-layer="3" window-active="true" window-focused="false" window-bounds="[0,0][390,844]"/></hierarchy>';
167175

168176
assert.deepEqual(Array.from(androidUiNodes(xml)), [
169177
{
@@ -177,6 +185,7 @@ test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
177185
clickable: false,
178186
enabled: true,
179187
visibleToUser: true,
188+
drawingOrder: 4,
180189
focusable: true,
181190
focused: true,
182191
password: true,
@@ -272,7 +281,7 @@ test('parseUiHierarchy excludes Android nodes that are not visible to the user',
272281
);
273282
});
274283

275-
test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapshots', () => {
284+
test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => {
276285
const xml = `<hierarchy>
277286
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
278287
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="false"/>
@@ -281,8 +290,114 @@ test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapsho
281290

282291
const result = parseUiHierarchy(xml, 800, { raw: true });
283292
assert.equal(result.nodes[0]!.visibleToUser, true);
284-
assert.equal(result.nodes[1]!.label, 'Hidden drawer action');
285-
assert.equal(result.nodes[1]!.visibleToUser, false);
293+
assert.equal(
294+
result.nodes.some((node) => node.label === 'Hidden drawer action'),
295+
false,
296+
);
297+
});
298+
299+
test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => {
300+
const xml = `<hierarchy>
301+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">
302+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" enabled="true" visible-to-user="false">
303+
<node class="android.widget.Button" text="Hidden drawer action" bounds="[10,80][200,120]" clickable="true" enabled="true" visible-to-user="true"/>
304+
</node>
305+
</node>
306+
</hierarchy>`;
307+
308+
const result = parseUiHierarchy(xml, 800, { raw: true });
309+
assert.equal(
310+
result.nodes.some((node) => node.label === 'Hidden drawer action'),
311+
false,
312+
);
313+
});
314+
315+
test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => {
316+
const xml = `<hierarchy>
317+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
318+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
319+
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
320+
</node>
321+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
322+
<node class="android.widget.ScrollView" bounds="[0,120][300,844]" scrollable="true" clickable="true" enabled="true" visible-to-user="true" drawing-order="1">
323+
<node class="android.widget.Button" text="Hidden drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
324+
</node>
325+
</node>
326+
</node>
327+
</hierarchy>`;
328+
329+
const result = parseUiHierarchy(xml, 800, { raw: true });
330+
assert.equal(
331+
result.nodes.some((node) => node.label === 'Foreground action'),
332+
true,
333+
);
334+
assert.equal(
335+
result.nodes.some((node) => node.label === 'Hidden drawer action'),
336+
false,
337+
);
338+
});
339+
340+
test('parseUiHierarchy keeps visible side-by-side drawer and content subtrees', () => {
341+
const xml = `<hierarchy>
342+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
343+
<node class="android.view.ViewGroup" bounds="[0,0][120,844]" visible-to-user="true" drawing-order="2">
344+
<node class="android.widget.Button" text="Visible drawer action" bounds="[0,220][110,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
345+
</node>
346+
<node class="android.view.ViewGroup" bounds="[120,0][390,844]" visible-to-user="true" drawing-order="1">
347+
<node class="android.widget.Button" text="Visible content action" bounds="[150,420][366,480]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
348+
</node>
349+
</node>
350+
</hierarchy>`;
351+
352+
const result = parseUiHierarchy(xml, 800, { raw: true });
353+
assert.equal(
354+
result.nodes.some((node) => node.label === 'Visible drawer action'),
355+
true,
356+
);
357+
assert.equal(
358+
result.nodes.some((node) => node.label === 'Visible content action'),
359+
true,
360+
);
361+
});
362+
363+
test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unavailable', () => {
364+
const xml = `<hierarchy>
365+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true">
366+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
367+
<node class="android.widget.Button" text="Foreground action" bounds="[24,420][366,480]" clickable="true" enabled="true" visible-to-user="true"/>
368+
</node>
369+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true">
370+
<node class="android.widget.Button" text="Legacy drawer action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true"/>
371+
</node>
372+
</node>
373+
</hierarchy>`;
374+
375+
const result = parseUiHierarchy(xml, 800, { raw: true });
376+
assert.equal(
377+
result.nodes.some((node) => node.label === 'Foreground action'),
378+
true,
379+
);
380+
assert.equal(
381+
result.nodes.some((node) => node.label === 'Legacy drawer action'),
382+
true,
383+
);
384+
});
385+
386+
test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible overlays', () => {
387+
const xml = `<hierarchy>
388+
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
389+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2"/>
390+
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
391+
<node class="android.widget.Button" text="Still visible action" bounds="[0,220][280,280]" clickable="true" enabled="true" visible-to-user="true" drawing-order="1"/>
392+
</node>
393+
</node>
394+
</hierarchy>`;
395+
396+
const result = parseUiHierarchy(xml, 800, { raw: true });
397+
assert.equal(
398+
result.nodes.some((node) => node.label === 'Still visible action'),
399+
true,
400+
);
286401
});
287402

288403
test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {

‎src/platforms/android/ui-hierarchy.ts‎

Lines changed: 120 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export type AndroidUiNodeMetadata = {
1717
clickable?: boolean;
1818
enabled?: boolean;
1919
visibleToUser?: boolean;
20+
drawingOrder?: number;
2021
focusable?: boolean;
2122
focused?: boolean;
2223
password?: boolean;
@@ -244,6 +245,7 @@ function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
244245
focused: boolAttr('focused'),
245246
password: boolAttr('password'),
246247
...optionalBoolAttr('visibleToUser', 'visible-to-user'),
248+
...optionalNumberAttr('drawingOrder', 'drawing-order'),
247249
...optionalBoolAttr('scrollable', 'scrollable'),
248250
...optionalBoolAttr('canScrollForward', 'can-scroll-forward'),
249251
...optionalBoolAttr('canScrollBackward', 'can-scroll-backward'),
@@ -383,7 +385,7 @@ function readXmlAttr(attrs: Map<string, string>, name: string): string | null {
383385

384386
function parseBounds(bounds: string | null): Rect | undefined {
385387
if (!bounds) return undefined;
386-
const match = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(bounds);
388+
const match = /\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/.exec(bounds);
387389
if (!match) return undefined;
388390
const x1 = Number(match[1]);
389391
const y1 = Number(match[2]);
@@ -401,6 +403,7 @@ export type AndroidUiHierarchy = {
401403
rect?: Rect;
402404
enabled?: boolean;
403405
visibleToUser?: boolean;
406+
drawingOrder?: number;
404407
hittable?: boolean;
405408
depth: number;
406409
parentIndex?: number;
@@ -428,6 +431,15 @@ type AndroidNodeInclusionInfo = {
428431
isVisual: boolean;
429432
};
430433

434+
type AndroidTreePruneState = {
435+
agentVisibleContentMemo: WeakMap<AndroidNode, boolean>;
436+
};
437+
438+
type AndroidCoveringCandidate = AndroidNode & {
439+
rect: Rect;
440+
drawingOrder: number;
441+
};
442+
431443
const ANDROID_WINDOW_TYPE_APPLICATION = 1;
432444

433445
export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
@@ -461,6 +473,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
461473
rect: attrs.rect,
462474
enabled: attrs.enabled,
463475
visibleToUser: attrs.visibleToUser,
476+
drawingOrder: attrs.drawingOrder,
464477
hittable: attrs.clickable ?? attrs.focusable,
465478
scrollable: attrs.scrollable,
466479
canScrollForward: attrs.canScrollForward,
@@ -481,11 +494,117 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
481494
}
482495
match = tokenRegex.exec(xml);
483496
}
497+
// Raw Android snapshots are uncollapsed, but still agent-visible. The helper can expose
498+
// aria-hidden/no-hide-descendants children, so prune nodes Android marks hidden to users.
499+
pruneAndroidInvisibleSubtrees(root);
484500
discardInactiveAndroidApplicationWindows(root);
501+
// UiAutomation can expose covered React Native navigation surfaces in the same accessibility
502+
// window. If a higher drawing-order sibling covers them, agents should see the foreground surface.
503+
pruneAndroidCoveredSubtrees(root, { agentVisibleContentMemo: new WeakMap() });
485504
applyAndroidScrollActionHints(root);
486505
return root;
487506
}
488507

508+
function pruneAndroidInvisibleSubtrees(node: AndroidNode): void {
509+
let keptCount = 0;
510+
for (const child of node.children) {
511+
if (child.visibleToUser === false) continue;
512+
pruneAndroidInvisibleSubtrees(child);
513+
node.children[keptCount] = child;
514+
keptCount += 1;
515+
}
516+
if (keptCount < node.children.length) {
517+
node.children.length = keptCount;
518+
}
519+
}
520+
521+
function pruneAndroidCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneState): void {
522+
for (const child of node.children) {
523+
pruneAndroidCoveredSubtrees(child, state);
524+
}
525+
if (node.children.length < 2) {
526+
return;
527+
}
528+
const siblings = node.children;
529+
const coveringCandidates = siblings.filter((sibling) => canCoverSibling(sibling, state));
530+
if (coveringCandidates.length === 0) return;
531+
node.children = siblings.filter(
532+
(child) => !isCoveredByHigherDrawingOrderSibling(child, coveringCandidates),
533+
);
534+
}
535+
536+
function isCoveredByHigherDrawingOrderSibling(
537+
node: AndroidNode,
538+
coveringCandidates: AndroidCoveringCandidate[],
539+
): boolean {
540+
if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) {
541+
return false;
542+
}
543+
544+
for (const sibling of coveringCandidates) {
545+
if (sibling === node || sibling.drawingOrder <= node.drawingOrder) {
546+
continue;
547+
}
548+
if (rectCoverage(sibling.rect, node.rect) >= 0.9) {
549+
return true;
550+
}
551+
}
552+
return false;
553+
}
554+
555+
function canCoverSibling(
556+
node: AndroidNode,
557+
state: AndroidTreePruneState,
558+
): node is AndroidCoveringCandidate {
559+
return (
560+
node.visibleToUser !== false &&
561+
node.drawingOrder !== undefined &&
562+
hasPositiveRect(node) &&
563+
hasAgentVisibleContent(node, state)
564+
);
565+
}
566+
567+
function hasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
568+
const cached = state.agentVisibleContentMemo.get(node);
569+
if (cached !== undefined) return cached;
570+
571+
const result = computeHasAgentVisibleContent(node, state);
572+
state.agentVisibleContentMemo.set(node, result);
573+
return result;
574+
}
575+
576+
function computeHasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
577+
if (node.visibleToUser === false) return false;
578+
if (node.hittable) return true;
579+
const label = node.label?.trim() ?? '';
580+
if (label && !isGenericAndroidId(label)) return true;
581+
const identifier = node.identifier?.trim() ?? '';
582+
if (identifier && !isGenericAndroidId(identifier)) return true;
583+
return node.children.some((child) => hasAgentVisibleContent(child, state));
584+
}
585+
586+
function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } {
587+
return Boolean(node.rect && node.rect.width > 0 && node.rect.height > 0);
588+
}
589+
590+
function rectCoverage(coveringRect: Rect, targetRect: Rect): number {
591+
const targetArea = targetRect.width * targetRect.height;
592+
if (targetArea <= 0) return 0;
593+
return intersectionArea(coveringRect, targetRect) / targetArea;
594+
}
595+
596+
function intersectionArea(left: Rect, right: Rect): number {
597+
const xOverlap = Math.max(
598+
0,
599+
Math.min(left.x + left.width, right.x + right.width) - Math.max(left.x, right.x),
600+
);
601+
const yOverlap = Math.max(
602+
0,
603+
Math.min(left.y + left.height, right.y + right.height) - Math.max(left.y, right.y),
604+
);
605+
return xOverlap * yOverlap;
606+
}
607+
489608
function applyAndroidScrollActionHints(root: AndroidUiHierarchy): void {
490609
const stack = [...root.children];
491610
while (stack.length > 0) {

0 commit comments

Comments
 (0)