Skip to content

Commit 5df37ec

Browse files
authored
fix: improve android fill verification diagnostics (#495)
1 parent 25d7289 commit 5df37ec

11 files changed

Lines changed: 791 additions & 112 deletions

File tree

examples/test-app/src/lab-state.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ const initialFormState: CheckoutFormState = {
77
name: '',
88
email: '',
99
phone: '',
10+
imeCaptureTarget: '',
1011
notes: '',
1112
shipping: 'Delivery',
1213
payment: 'Card',

examples/test-app/src/screens/FormScreen.tsx

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export interface CheckoutFormState {
1414
name: string;
1515
email: string;
1616
phone: string;
17+
imeCaptureTarget: string;
1718
notes: string;
1819
shipping: 'Delivery' | 'Pickup';
1920
payment: 'Card' | 'Cash';
@@ -132,6 +133,32 @@ export function FormScreen(props: FormScreenProps) {
132133
/>
133134
</SectionCard>
134135

136+
<SectionCard
137+
subtitle="A fixture for Android cases where Gboard handwriting owns the focused input."
138+
title="Android IME capture"
139+
testID="android-ime-capture-fixture"
140+
>
141+
{/* SkillGym fixture: static diagnostic copy, not live state. */}
142+
<TextField
143+
accessibilityLabel="Android IME target field"
144+
autoCapitalize="none"
145+
label="Android IME target field"
146+
onChangeText={(value) => props.onChange('imeCaptureTarget', value)}
147+
placeholder="Search term"
148+
testID="field-ime-capture-target"
149+
value={props.form.imeCaptureTarget}
150+
/>
151+
<View style={styles.diagnosticBlock} testID="ime-capture-diagnostic">
152+
<Text style={styles.diagnosticText}>
153+
Android fill input was captured by the active keyboard instead of the app field
154+
</Text>
155+
<Text style={styles.diagnosticMeta}>targetInput id="field-ime-capture-target"</Text>
156+
<Text style={styles.diagnosticMeta}>
157+
actualInput packageName="com.google.android.inputmethod.latin" inputMethodOwned=true
158+
</Text>
159+
</View>
160+
</SectionCard>
161+
135162
<SectionCard
136163
subtitle="These button groups are stable selector targets."
137164
title="Delivery choices"
@@ -231,6 +258,25 @@ function createStyles(colors: AppColors) {
231258
flexWrap: 'wrap',
232259
gap: 8,
233260
},
261+
diagnosticBlock: {
262+
backgroundColor: colors.cardStrong,
263+
borderColor: colors.line,
264+
borderRadius: 8,
265+
borderWidth: StyleSheet.hairlineWidth,
266+
gap: 6,
267+
padding: 12,
268+
},
269+
diagnosticMeta: {
270+
color: colors.textSoft,
271+
fontFamily: 'monospace',
272+
fontSize: 12,
273+
lineHeight: 18,
274+
},
275+
diagnosticText: {
276+
color: colors.text,
277+
fontSize: 14,
278+
lineHeight: 20,
279+
},
234280
checkboxRow: {
235281
alignItems: 'center',
236282
flexDirection: 'row',
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { buildFillFailureDetails } from '../fill-diagnostics.ts';
4+
5+
test('buildFillFailureDetails redacts masked expected and node text', () => {
6+
const details = buildFillFailureDetails('Secret123', {
7+
ok: false,
8+
actual: 'secret-value',
9+
reason: 'masked_unverified',
10+
masked: true,
11+
targetInput: {
12+
text: 'secret-value',
13+
password: true,
14+
focused: true,
15+
},
16+
actualInput: {
17+
text: '••••••',
18+
focused: true,
19+
},
20+
});
21+
22+
assert.equal(details.expected, undefined);
23+
assert.equal(details.expectedLength, 9);
24+
assert.equal(details.actual, null);
25+
assert.equal(details.actualLength, 12);
26+
assert.equal(details.targetInput?.text, null);
27+
assert.equal(details.targetInput?.textRedacted, true);
28+
assert.equal(details.actualInput?.text, null);
29+
assert.equal(details.actualInput?.textRedacted, true);
30+
assert.doesNotMatch(JSON.stringify(details), /Secret123|secret-value|/);
31+
});
32+
33+
test('buildFillFailureDetails keeps non-masked text diagnostics visible', () => {
34+
const details = buildFillFailureDetails('search term', {
35+
ok: false,
36+
actual: 'search',
37+
reason: 'text_mismatch',
38+
targetInput: { text: 'Search Products', focused: false },
39+
actualInput: { text: 'search', focused: true },
40+
});
41+
42+
assert.equal(details.expected, 'search term');
43+
assert.equal(details.actual, 'search');
44+
assert.equal(details.targetInput?.text, 'Search Products');
45+
assert.equal(details.actualInput?.text, 'search');
46+
});
47+
48+
test('buildFillFailureDetails infers sensitivity from password nodes', () => {
49+
const details = buildFillFailureDetails('Secret123', {
50+
ok: false,
51+
actual: 'secret-value',
52+
reason: 'text_mismatch',
53+
targetInput: { text: 'secret-value', password: true },
54+
actualInput: { text: 'secret-value', password: true },
55+
});
56+
57+
assert.equal(details.expected, undefined);
58+
assert.equal(details.expectedLength, 9);
59+
assert.equal(details.actual, null);
60+
assert.equal(details.actualInput?.textRedacted, true);
61+
assert.doesNotMatch(JSON.stringify(details), /Secret123|secret-value/);
62+
});
63+
64+
test('buildFillFailureDetails redacts common masked field glyphs', () => {
65+
for (const actual of ['••••', '****', '●●●']) {
66+
const details = buildFillFailureDetails('Secret123', {
67+
ok: false,
68+
actual,
69+
reason: 'masked_unverified',
70+
targetInput: { text: 'Search Products' },
71+
actualInput: { text: actual, focused: true },
72+
});
73+
74+
assert.equal(details.expected, undefined);
75+
assert.equal(details.actual, null);
76+
assert.equal(details.actualInput?.textRedacted, true);
77+
assert.doesNotMatch(JSON.stringify(details), /Secret123|\*||/);
78+
}
79+
});
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts';
4+
import { AppError } from '../../../utils/errors.ts';
5+
import { fillAndroid } from '../index.ts';
6+
import { withAndroidAdbProvider, type AndroidAdbExecutor } from '../adb-executor.ts';
7+
import {
8+
androidFillFailureDetails,
9+
androidFillFailureMessage,
10+
readAndroidTextAtPointInHierarchy,
11+
verifyAndroidFilledTextInHierarchy,
12+
} from '../fill-verification.ts';
13+
14+
test('fillAndroid reports when the IME captures input instead of the app field', async () => {
15+
const calls: string[][] = [];
16+
let imeText = '';
17+
await withFillAdb(
18+
async (args) => {
19+
calls.push(args);
20+
if (isTextInput(args)) imeText = args[3] ?? '';
21+
return adbResult(args[0] === 'exec-out' ? imeCaptureHierarchy(imeText) : '');
22+
},
23+
async () => {
24+
await assert.rejects(
25+
() => fillAndroid(ANDROID_EMULATOR, 10, 10, 'chips'),
26+
(error: unknown) => {
27+
assert.ok(error instanceof AppError);
28+
assert.equal(error.code, 'COMMAND_FAILED');
29+
assert.match(error.message, /captured by the active keyboard/i);
30+
assert.equal(error.details?.failureReason, 'ime_capture');
31+
assert.equal(inputDetails(error, 'actualInput')?.resourceId, IME_RESOURCE_ID);
32+
assert.equal(
33+
inputDetails(error, 'targetInput')?.resourceId,
34+
'com.example.shop:id/search',
35+
);
36+
return true;
37+
},
38+
);
39+
},
40+
);
41+
42+
assert.equal(
43+
calls.some((args) => args.join('\n') === 'shell\ncmd\nclipboard\nset\ntext'),
44+
false,
45+
);
46+
assert.equal(
47+
calls.some((args) => args.join('\n') === 'shell\ninput\nkeyevent\nKEYCODE_PASTE'),
48+
false,
49+
);
50+
});
51+
52+
test('verifyAndroidFilledTextInHierarchy accepts matching-length masked password verification', () => {
53+
const verification = verifyAndroidFilledTextInHierarchy(
54+
passwordHierarchy(maskBullets('Test@123')),
55+
10,
56+
10,
57+
'Test@123',
58+
);
59+
60+
assert.equal(verification.ok, true);
61+
assert.equal(verification.masked, true);
62+
});
63+
64+
test('fillAndroid accepts matching-length masked password verification', async () => {
65+
let typed = '';
66+
await withFillAdb(
67+
async (args) => {
68+
if (isDeleteKey(args)) typed = '';
69+
if (isTextInput(args)) typed = args[3] ?? '';
70+
return adbResult(args[0] === 'exec-out' ? passwordHierarchy(maskBullets(typed)) : '');
71+
},
72+
async () => {
73+
await fillAndroid(ANDROID_EMULATOR, 10, 10, 'Test@123');
74+
},
75+
);
76+
});
77+
78+
test('verifyAndroidFilledTextInHierarchy redacts masked password values on wrong-length failure', () => {
79+
const exposedPasswordValue = 'secret-value';
80+
const verification = verifyAndroidFilledTextInHierarchy(
81+
passwordHierarchy(exposedPasswordValue),
82+
10,
83+
10,
84+
'Test@123',
85+
);
86+
87+
assert.equal(verification.ok, false);
88+
assertMaskedPasswordFailure(
89+
androidFillFailureMessage(verification),
90+
androidFillFailureDetails('Test@123', verification),
91+
exposedPasswordValue,
92+
);
93+
});
94+
95+
test('readAndroidTextAtPointInHierarchy prefers focused edit text over point fallback', () => {
96+
assert.equal(readAndroidTextAtPointInHierarchy(focusedEditHierarchy(), 10, 10), 'focused value');
97+
});
98+
99+
const IME_RESOURCE_ID = 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated';
100+
101+
async function withFillAdb(exec: AndroidAdbExecutor, fn: () => Promise<void>): Promise<void> {
102+
await withAndroidAdbProvider({ exec }, { serial: ANDROID_EMULATOR.id }, fn);
103+
}
104+
105+
function adbResult(stdout: string) {
106+
return { stdout, stderr: '', exitCode: 0 };
107+
}
108+
109+
function isTextInput(args: string[]): boolean {
110+
return args[0] === 'shell' && args[1] === 'input' && args[2] === 'text';
111+
}
112+
113+
function isDeleteKey(args: string[]): boolean {
114+
return args.join('\n') === 'shell\ninput\nkeyevent\nKEYCODE_DEL';
115+
}
116+
117+
function inputDetails(error: AppError, key: 'actualInput' | 'targetInput') {
118+
return error.details?.[key] as Record<string, unknown> | null | undefined;
119+
}
120+
121+
function assertMaskedPasswordFailure(
122+
message: string,
123+
details: Record<string, unknown>,
124+
exposedPasswordValue: string,
125+
): void {
126+
assert.match(message, /could not confirm masked text value/i);
127+
assert.equal(details.failureReason, 'masked_unverified');
128+
assert.equal(details.masked, true);
129+
assert.equal(details.expected, undefined);
130+
assert.equal(details.expectedLength, 8);
131+
assert.equal(details.actual, null);
132+
assert.equal(details.actualLength, exposedPasswordValue.length);
133+
assert.equal(detailsInput(details, 'actualInput')?.text, null);
134+
assert.equal(detailsInput(details, 'actualInput')?.textRedacted, true);
135+
assert.equal(detailsInput(details, 'targetInput')?.text, null);
136+
assert.doesNotMatch(JSON.stringify(details), /Test@123|secret-value/);
137+
}
138+
139+
function detailsInput(details: Record<string, unknown>, key: 'actualInput' | 'targetInput') {
140+
return details[key] as Record<string, unknown> | null | undefined;
141+
}
142+
143+
function imeCaptureHierarchy(imeText: string): string {
144+
return `<?xml version="1.0" encoding="UTF-8"?><hierarchy>
145+
<node package="com.example.shop" class="android.widget.EditText" text="Search Products" resource-id="com.example.shop:id/search" focused="false" bounds="[0,0][300,100]"/>
146+
<node package="com.google.android.inputmethod.latin" class="android.widget.EditText" text="${imeText}" resource-id="${IME_RESOURCE_ID}" focused="true" bounds="[0,700][300,800]"/>
147+
</hierarchy>`;
148+
}
149+
150+
function passwordHierarchy(mask: string): string {
151+
return `<?xml version="1.0" encoding="UTF-8"?><hierarchy><node package="com.example" class="android.widget.EditText" text="${mask}" password="true" focused="true" bounds="[0,0][200,100]"/></hierarchy>`;
152+
}
153+
154+
function focusedEditHierarchy(): string {
155+
return `<?xml version="1.0" encoding="UTF-8"?><hierarchy>
156+
<node package="com.example" class="android.widget.TextView" text="point fallback" focused="false" bounds="[0,0][200,100]"/>
157+
<node package="com.example" class="android.widget.EditText" text="focused value" focused="true" bounds="[300,300][500,400]"/>
158+
</hierarchy>`;
159+
}
160+
161+
function maskBullets(value: string): string {
162+
return Array.from(value)
163+
.map(() => '&#8226;')
164+
.join('');
165+
}

0 commit comments

Comments
 (0)