Skip to content

fix(android): parse uiautomator XML attributes with single or double quotes #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thymikee merged 3 commits into from
Feb 10, 2026
Merged

fix(android): parse uiautomator XML attributes with single or double quotes #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5668ae0
fix(android): parse uiautomator XML attributes with single or double …
Feb 9, 2026
c8309e1
Merge branch 'callstackincubator:main' into main
XiaoPengYouCode Feb 10, 2026
510a0ee
fix(android): harden hierarchy parser and tighten tests
thymikee Feb 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"prepack": "pnpm build:node && pnpm build:axsnapshot",
"typecheck": "tsc -p tsconfig.json",
"test": "node --test",
"test:unit": "node --test src/core/__tests__/*.test.ts src/daemon/__tests__/*.test.ts src/daemon/handlers/__tests__/*.test.ts",
"test:unit": "node --test src/core/__tests__/*.test.ts src/daemon/__tests__/*.test.ts src/daemon/handlers/__tests__/*.test.ts src/platforms/**/__tests__/*.test.ts",
"test:smoke": "node --test test/integration/smoke-*.test.ts",
"test:integration": "node --test test/integration/*.test.ts"
},
Expand Down
74 changes: 74 additions & 0 deletions src/platforms/android/__tests__/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { findBounds, parseUiHierarchy } from '../ui-hierarchy.ts';

test('parseUiHierarchy reads double-quoted Android node attributes', () => {
const xml =
'<hierarchy><node class="android.widget.TextView" text="Hello" content-desc="Greeting" resource-id="com.demo:id/title" bounds="[10,20][110,60]" clickable="true" enabled="true"/></hierarchy>';

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(result.nodes.length, 1);
assert.equal(result.nodes[0].value, 'Hello');
assert.equal(result.nodes[0].label, 'Hello');
assert.equal(result.nodes[0].identifier, 'com.demo:id/title');
assert.deepEqual(result.nodes[0].rect, { x: 10, y: 20, width: 100, height: 40 });
assert.equal(result.nodes[0].hittable, true);
assert.equal(result.nodes[0].enabled, true);
});

test('parseUiHierarchy reads single-quoted Android node attributes', () => {
const xml =
"<hierarchy><node class='android.widget.TextView' text='Hello' content-desc='Greeting' resource-id='com.demo:id/title' bounds='[10,20][110,60]' clickable='true' enabled='true'/></hierarchy>";

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(result.nodes.length, 1);
assert.equal(result.nodes[0].value, 'Hello');
assert.equal(result.nodes[0].label, 'Hello');
assert.equal(result.nodes[0].identifier, 'com.demo:id/title');
assert.deepEqual(result.nodes[0].rect, { x: 10, y: 20, width: 100, height: 40 });
assert.equal(result.nodes[0].hittable, true);
assert.equal(result.nodes[0].enabled, true);
});

test('parseUiHierarchy supports mixed quote styles in one node', () => {
const xml =
'<hierarchy><node class="android.widget.TextView" text=\'Hello\' content-desc="Greeting" resource-id=\'com.demo:id/title\' bounds="[10,20][110,60]"/></hierarchy>';

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(result.nodes.length, 1);
assert.equal(result.nodes[0].value, 'Hello');
assert.equal(result.nodes[0].label, 'Hello');
assert.equal(result.nodes[0].identifier, 'com.demo:id/title');
});

test('findBounds supports single and double quoted attributes', () => {
const xml = [
'<hierarchy>',
'<node text="Nothing" content-desc="Irrelevant" bounds="[0,0][10,10]"/>',
"<node text='Target from single quote' content-desc='Alt single' bounds='[100,200][300,500]'/>",
'<node text="Target from double quote" content-desc="Alt double" bounds="[50,50][150,250]"/>',
'</hierarchy>',
].join('');

assert.deepEqual(findBounds(xml, 'single quote'), { x: 200, y: 350 });
assert.deepEqual(findBounds(xml, 'alt double'), { x: 100, y: 150 });
});

test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
const xml =
"<hierarchy><node class='android.widget.TextView' hint-text='Spoofed' text='Actual' bounds='[10,20][110,60]'/></hierarchy>";

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(result.nodes.length, 1);
assert.equal(result.nodes[0].value, 'Actual');
});

test('findBounds ignores bounds-like fragments inside other attribute values', () => {
const xml = [
'<hierarchy>',
"<node text='Target' content-desc=\"metadata bounds='[900,900][1000,1000]'\" bounds='[100,200][300,500]'/>",
'</hierarchy>',
].join('');

assert.deepEqual(findBounds(xml, 'target'), { x: 200, y: 350 });
});
291 changes: 2 additions & 289 deletions src/platforms/android/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ import { runCmd, whichCmd } from '../../utils/exec.ts';
import { withRetry } from '../../utils/retry.ts';
import { AppError } from '../../utils/errors.ts';
import type { DeviceInfo } from '../../utils/device.ts';
import type { RawSnapshotNode, Rect, SnapshotOptions } from '../../utils/snapshot.ts';
import type { RawSnapshotNode, SnapshotOptions } from '../../utils/snapshot.ts';
import { waitForAndroidBoot } from './devices.ts';
import { findBounds, parseBounds, parseUiHierarchy, readNodeAttributes } from './ui-hierarchy.ts';

const ALIASES: Record<string, { type: 'intent' | 'package'; value: string }> = {
settings: { type: 'intent', value: 'android.settings.SETTINGS' },
Expand Down Expand Up @@ -623,291 +624,3 @@ async function sleep(ms: number): Promise<void> {
function clampCount(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}

function findBounds(xml: string, query: string): { x: number; y: number } | null {
const q = query.toLowerCase();
const nodeRegex = /<node[^>]+>/g;
let match = nodeRegex.exec(xml);
while (match) {
const node = match[0];
const textMatch = /text="([^"]*)"/.exec(node);
const descMatch = /content-desc="([^"]*)"/.exec(node);
const textVal = (textMatch?.[1] ?? '').toLowerCase();
const descVal = (descMatch?.[1] ?? '').toLowerCase();
if (textVal.includes(q) || descVal.includes(q)) {
const boundsMatch = /bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"/.exec(node);
if (boundsMatch) {
const x1 = Number(boundsMatch[1]);
const y1 = Number(boundsMatch[2]);
const x2 = Number(boundsMatch[3]);
const y2 = Number(boundsMatch[4]);
return { x: Math.floor((x1 + x2) / 2), y: Math.floor((y1 + y2) / 2) };
}
return { x: 0, y: 0 };
}
match = nodeRegex.exec(xml);
}
return null;
}

function parseUiHierarchy(
xml: string,
maxNodes: number,
options: SnapshotOptions,
): { nodes: RawSnapshotNode[]; truncated?: boolean } {
const tree = parseUiHierarchyTree(xml);
const nodes: RawSnapshotNode[] = [];
let truncated = false;
const maxDepth = options.depth ?? Number.POSITIVE_INFINITY;
const scopedRoot = options.scope ? findScopeNode(tree, options.scope) : null;
const roots = scopedRoot ? [scopedRoot] : tree.children;

const interactiveDescendantMemo = new Map<AndroidNode, boolean>();
const hasInteractiveDescendant = (node: AndroidNode): boolean => {
const cached = interactiveDescendantMemo.get(node);
if (cached !== undefined) return cached;
for (const child of node.children) {
if (child.hittable || hasInteractiveDescendant(child)) {
interactiveDescendantMemo.set(node, true);
return true;
}
}
interactiveDescendantMemo.set(node, false);
return false;
};

const walk = (
node: AndroidNode,
depth: number,
parentIndex?: number,
ancestorHittable: boolean = false,
ancestorCollection: boolean = false,
) => {
if (nodes.length >= maxNodes) {
truncated = true;
return;
}
if (depth > maxDepth) return;

const include = options.raw
? true
: shouldIncludeAndroidNode(
node,
options,
ancestorHittable,
hasInteractiveDescendant(node),
ancestorCollection,
);
let currentIndex = parentIndex;
if (include) {
currentIndex = nodes.length;
nodes.push({
index: currentIndex,
type: node.type ?? undefined,
label: node.label ?? undefined,
value: node.value ?? undefined,
identifier: node.identifier ?? undefined,
rect: node.rect,
enabled: node.enabled,
hittable: node.hittable,
depth,
parentIndex,
});
}
const nextAncestorHittable = ancestorHittable || Boolean(node.hittable);
const nextAncestorCollection = ancestorCollection || isCollectionContainerType(node.type);
for (const child of node.children) {
walk(child, depth + 1, currentIndex, nextAncestorHittable, nextAncestorCollection);
if (truncated) return;
}
};

for (const root of roots) {
walk(root, 0, undefined, false, false);
if (truncated) break;
}

return truncated ? { nodes, truncated } : { nodes };
}

function readNodeAttributes(node: string): {
text: string | null;
desc: string | null;
resourceId: string | null;
className: string | null;
bounds: string | null;
clickable?: boolean;
enabled?: boolean;
focusable?: boolean;
focused?: boolean;
} {
const getAttr = (name: string): string | null => {
const regex = new RegExp(`${name}="([^"]*)"`);
const match = regex.exec(node);
return match ? match[1] : null;
};
const boolAttr = (name: string): boolean | undefined => {
const raw = getAttr(name);
if (raw === null) return undefined;
return raw === 'true';
};
return {
text: getAttr('text'),
desc: getAttr('content-desc'),
resourceId: getAttr('resource-id'),
className: getAttr('class'),
bounds: getAttr('bounds'),
clickable: boolAttr('clickable'),
enabled: boolAttr('enabled'),
focusable: boolAttr('focusable'),
focused: boolAttr('focused'),
};
}

function parseBounds(bounds: string | null): Rect | undefined {
if (!bounds) return undefined;
const match = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(bounds);
if (!match) return undefined;
const x1 = Number(match[1]);
const y1 = Number(match[2]);
const x2 = Number(match[3]);
const y2 = Number(match[4]);
return { x: x1, y: y1, width: Math.max(0, x2 - x1), height: Math.max(0, y2 - y1) };
}

type AndroidNode = {
type: string | null;
label: string | null;
value: string | null;
identifier: string | null;
rect?: Rect;
enabled?: boolean;
hittable?: boolean;
depth: number;
parentIndex?: number;
children: AndroidNode[];
};

function parseUiHierarchyTree(xml: string): AndroidNode {
const root: AndroidNode = {
type: null,
label: null,
value: null,
identifier: null,
depth: -1,
children: [],
};
const stack: AndroidNode[] = [root];
const tokenRegex = /<node\b[^>]*>|<\/node>/g;
let match = tokenRegex.exec(xml);
while (match) {
const token = match[0];
if (token.startsWith('</node')) {
if (stack.length > 1) stack.pop();
match = tokenRegex.exec(xml);
continue;
}
const attrs = readNodeAttributes(token);
const rect = parseBounds(attrs.bounds);
const parent = stack[stack.length - 1];
const node: AndroidNode = {
type: attrs.className,
label: attrs.text || attrs.desc,
value: attrs.text,
identifier: attrs.resourceId,
rect,
enabled: attrs.enabled,
hittable: attrs.clickable ?? attrs.focusable,
depth: parent.depth + 1,
parentIndex: undefined,
children: [],
};
parent.children.push(node);
if (!token.endsWith('/>')) {
stack.push(node);
}
match = tokenRegex.exec(xml);
}
return root;
}

function shouldIncludeAndroidNode(
node: AndroidNode,
options: SnapshotOptions,
ancestorHittable: boolean,
descendantHittable: boolean,
ancestorCollection: boolean,
): boolean {
const type = normalizeAndroidType(node.type);
const hasText = Boolean(node.label && node.label.trim().length > 0);
const hasId = Boolean(node.identifier && node.identifier.trim().length > 0);
const hasMeaningfulText = hasText && !isGenericAndroidId(node.label ?? '');
const hasMeaningfulId = hasId && !isGenericAndroidId(node.identifier ?? '');
const isStructural = isStructuralAndroidType(type);
const isVisual = type === 'imageview' || type === 'imagebutton';
if (options.interactiveOnly) {
if (node.hittable) return true;
// Keep text proxies for tappable rows while dropping structural noise.
const proxyCandidate = hasMeaningfulText || hasMeaningfulId;
if (!proxyCandidate) return false;
if (isVisual) return false;
if (isStructural && !ancestorCollection) return false;
return ancestorHittable || descendantHittable || ancestorCollection;
}
if (options.compact) {
return hasMeaningfulText || hasMeaningfulId || Boolean(node.hittable);
}
if (isStructural || isVisual) {
if (node.hittable) return true;
if (hasMeaningfulText) return true;
if (hasMeaningfulId && descendantHittable) return true;
return descendantHittable;
}
return true;
}

function isCollectionContainerType(type: string | null): boolean {
if (!type) return false;
const normalized = normalizeAndroidType(type);
return (
normalized.includes('recyclerview') ||
normalized.includes('listview') ||
normalized.includes('gridview')
);
}

function normalizeAndroidType(type: string | null): string {
if (!type) return '';
return type.toLowerCase();
}

function isStructuralAndroidType(type: string): boolean {
const short = type.split('.').pop() ?? type;
return (
short.includes('layout') ||
short === 'viewgroup' ||
short === 'view'
);
}

function isGenericAndroidId(value: string): boolean {
const trimmed = value.trim();
if (!trimmed) return false;
return /^[\w.]+:id\/[\w.-]+$/i.test(trimmed);
}

function findScopeNode(root: AndroidNode, scope: string): AndroidNode | null {
const query = scope.toLowerCase();
const stack: AndroidNode[] = [...root.children];
while (stack.length > 0) {
const node = stack.shift() as AndroidNode;
const label = node.label?.toLowerCase() ?? '';
const value = node.value?.toLowerCase() ?? '';
const identifier = node.identifier?.toLowerCase() ?? '';
if (label.includes(query) || value.includes(query) || identifier.includes(query)) {
return node;
}
stack.push(...node.children);
}
return null;
}
Loading
Loading