feat(keyboard-guard): in-runner keyboard-occlusion guard for live device_* taps, iOS + Android (#370)

.changeset/370-keyboard-occlusion-guard-phase2.md

@@ -0,0 +1,6 @@
+---
+"rn-dev-agent-cdp": patch
+"rn-dev-agent-plugin": patch
+---
+
+feat(keyboard-guard): in-runner keyboard-occlusion guard for live `device_press`/`device_longpress` taps on iOS + Android (#370). Before a guarded tap, the runner probes for a visible software keyboard whose frame contains the tap point (containment on a sane rect — non-empty, min height 120pt iOS / 150px Android, so accessory bars don't false-trigger) and auto-dismisses first when occluded. Android dismissal is `pressBack` + a bounded `waitForIdle(1500)` (≈3.6s measured incl. bounded idle), gated on a TYPE_INPUT_METHOD window with sane bounds so it never navigates back otherwise — requires `FLAG_RETRIEVE_INTERACTIVE_WINDOWS`, now enabled at dispatcher init. iOS is verify-or-refuse: only the safe dismiss-control tap ("Hide keyboard"/"Dismiss keyboard"/"Done") is used, then re-verified; on iPhone standard QWERTY, which has no such control, the runner REFUSES the tap with `KEYBOARD_OCCLUDED … keyboardGuard=dismiss_failed` instead of tapping the keyboard, because XCTest's `swipeDown` on the keyboard triggers QuickPath slide-typing and corrupts the focused field (device-proven). Every guarded gesture returns `meta.keyboardGuard`: `"off" | "no_keyboard" | "not_occluded" | "dismissed"` (plus `dismiss_failed` inside the iOS refusal error). Opt out with `RN_KEYBOARD_GUARD=0`/`false`, resolved TS-side per command (`guardKeyboard` on the wire; absent → guard stays ON, so older clients keep guarding). Scope is command-handler tap/longPress only — `tapSeries`, by-text taps, element-center taps, the focus-tap inside type/fill, swipes/scrolls/drags, and `doubleTap` are explicitly unguarded. Follow-up #379 tracks a JS-first (`Keyboard.dismiss()`) auto-heal for the iOS refusal case; #378 tracks a pre-existing Android `foreground()` pre-flight stall surfaced (not fixed) during verification.

docs-site/src/content/docs/skills/rn-device-control.mdx

@@ -72,3 +72,24 @@ Decision table for build workflows:
 | adb shows "unauthorized" | Revoke USB debugging, reconnect, tap Allow |
 | Screenshot hangs | Wait for home screen, verify boot completed |
 | uiautomator dump fails | Wake screen with keyevent WAKEUP |
+
+### Keyboard-occlusion guard
+
+`device_press` and `device_longpress` guard against tapping through a software keyboard by default. Before the tap lands, the runner checks whether a visible keyboard's frame contains the tap point (a sanity-checked rect — min height 120pt iOS / 150px Android, so a predictive-text accessory bar doesn't count) and dismisses it first if so.
+
+| `meta.keyboardGuard` | Meaning |
+|---|---|
+| `off` | Guard disabled (`RN_KEYBOARD_GUARD=0`) |
+| `no_keyboard` | No keyboard was visible |
+| `not_occluded` | Keyboard visible but the tap point was clear of it |
+| `dismissed` | Keyboard was occluding the tap point and was dismissed first |
+
+Platform behavior differs because the safe dismissal mechanism differs:
+- **Android** — `pressBack` + a bounded wait for the UI to settle (~3.6s measured). Only fires when a real input-method window with sane bounds contains the tap point, so it never triggers an unrelated back-navigation.
+- **iOS** — verify-or-refuse. The runner only ever taps a genuine dismiss control ("Hide keyboard" / "Dismiss keyboard" / "Done") and re-verifies the keyboard is gone. On the standard iPhone QWERTY keyboard, which has no such control, the tap is **refused** with a `KEYBOARD_OCCLUDED` error (`keyboardGuard=dismiss_failed`) instead of swiping the keyboard closed — a swipe-based dismiss triggers QuickPath slide-typing and corrupts whatever the keyboard was over.
+
+If you hit `KEYBOARD_OCCLUDED` on iOS: dismiss the keyboard yourself first (`device_fill`/`cdp_interact` type through the JS path and don't require the keyboard to be down, or tap a non-input area of the screen) and then retry the tap.
+
+Opt out entirely with `RN_KEYBOARD_GUARD=0` (or `false`) if the guard gets in the way of a flow you control precisely.
+
+Only `device_press`/`device_longpress`-style taps are guarded — `tapSeries`, text-based taps, swipes/scrolls/drags, and `doubleTap` are unaffected.

scripts/cdp-bridge/dist/runners/keyboard-guard.js

@@ -0,0 +1,32 @@
+export function resolveKeyboardGuard(env) {
+    const raw = (env.RN_KEYBOARD_GUARD ?? '').trim().toLowerCase();
+    return !(raw === '0' || raw === 'false');
+}
+const GUARDED_VERBS = new Set(['tap', 'press', 'longPress']);
+export function withKeyboardGuard(payload, verb, env) {
+    if (!GUARDED_VERBS.has(verb))
+        return payload;
+    return { ...payload, guardKeyboard: resolveKeyboardGuard(env) };
+}
+export function surfaceKeyboardGuard(result) {
+    const text = result.content?.[0]?.text;
+    if (typeof text !== 'string')
+        return result;
+    let envelope;
+    try {
+        envelope = JSON.parse(text);
+    }
+    catch {
+        return result;
+    }
+    const data = envelope.data;
+    const keyboardGuard = data?.keyboardGuard;
+    if (typeof keyboardGuard !== 'string')
+        return result;
+    const meta = envelope.meta ?? {};
+    envelope.meta = { ...meta, keyboardGuard };
+    return {
+        ...result,
+        content: [{ type: 'text', text: JSON.stringify(envelope) }],
+    };
+}

scripts/cdp-bridge/dist/runners/rn-android-runner-client.js

@@ -9,6 +9,7 @@ import { okResult, failResult } from '../utils.js';
 import { updateRefMapFromFlat, getCachedMetadata } from '../fast-runner-ref-map.js';
 import { findFreePort } from './free-port.js';
 import { join } from 'node:path';
+import { withKeyboardGuard } from './keyboard-guard.js';
 const execFileAsync = promisify(execFile);
 const DEFAULT_PORT = 22089;
 const READY_TIMEOUT_MS = 30_000;
@@ -468,7 +469,7 @@ export async function runAndroid(args) {
     let resp;
     try {
         await startAndroidRunner(args.deviceId, args.bundleId);
-        resp = await postCommand(body);
+        resp = await postCommand(withKeyboardGuard(body, args.command, process.env));
     }
     catch (err) {
         const m = errMessage(err);

scripts/cdp-bridge/dist/runners/rn-fast-runner-client.js

@@ -4,6 +4,7 @@ import { writeFileSync, unlinkSync, readFileSync, existsSync, readdirSync } from
 import { okResult, failResult } from '../utils.js';
 import { updateRefMapFromFlat, getCachedMetadata } from '../fast-runner-ref-map.js';
 import { isPortFree } from './free-port.js';
+import { withKeyboardGuard } from './keyboard-guard.js';
 const DEFAULT_PORT = 22088;
 const READY_TIMEOUT_MS = 30_000;
 // A cold `xcodebuild test` compiles the runner project before launching it; on a
@@ -508,7 +509,7 @@ export async function runIOS(args) {
         body.depth = args.depth;
     if (args.scope !== undefined)
         body.scope = args.scope;
-    const resp = await postCommand(body);
+    const resp = await postCommand(withKeyboardGuard(body, args.command, process.env));
     if (!resp.ok) {
         const message = resp.error?.message ?? 'runner returned !ok with no error';
         const code = resp.error?.code;

scripts/cdp-bridge/dist/tools/device-interact.js

@@ -3,6 +3,7 @@ import { promisify } from 'node:util';
 import { runNative, getActiveSession, clearActiveSession, getCachedScreenRect, getAdbSerial, cacheSnapshot, getCachedSnapshot, isSnapshotCacheValid, } from '../agent-device-wrapper.js';
 import { isFastRunnerAvailable, fastSwipe, stopFastRunner, } from '../runners/rn-fast-runner-client.js';
 import { stopAndroidRunner } from '../runners/rn-android-runner-client.js';
+import { surfaceKeyboardGuard } from '../runners/keyboard-guard.js';
 import { withSession } from '../utils.js';
 import { okResult, failResult, createStepTimer } from '../utils.js';
 import { runMaestroInline, yamlEscape } from '../maestro-invoke.js';
@@ -175,7 +176,7 @@ function runnerLeakFailResult(query, recoveryReason) {
 export async function pressCandidate(candidate, action) {
     const ref = candidate.ref.startsWith('@') ? candidate.ref : `@${candidate.ref}`;
     if (action === 'click') {
-        return runNative(['press', ref]);
+        return surfaceKeyboardGuard(await runNative(['press', ref]));
     }
     return okResult({ ref: candidate.ref, label: candidate.label, testID: candidate.testID });
 }
@@ -336,27 +337,27 @@ export function createDevicePressHandler() {
             cliArgs.push('--count', String(args.count));
         if (args.holdMs && args.holdMs > 0)
             cliArgs.push('--hold-ms', String(args.holdMs));
-        const result = await runNative(cliArgs);
+        const result = surfaceKeyboardGuard(await runNative(cliArgs));
         if (!result.isError && args.waitForFocusMs && args.waitForFocusMs > 0) {
             await new Promise((r) => setTimeout(r, args.waitForFocusMs));
         }
         return result;
     });
 }
 export function createDeviceLongPressHandler() {
-    return withSession((args) => {
+    return withSession(async (args) => {
         if (args.ref) {
             const ref = args.ref.startsWith('@') ? args.ref : `@${args.ref}`;
             const cliArgs = ['press', ref, '--hold-ms', String(args.durationMs ?? 1000)];
-            return runNative(cliArgs);
+            return surfaceKeyboardGuard(await runNative(cliArgs));
         }
         if (args.x != null && args.y != null) {
             const cliArgs = ['longpress', String(args.x), String(args.y)];
             if (args.durationMs)
                 cliArgs.push(String(args.durationMs));
-            return runNative(cliArgs);
+            return surfaceKeyboardGuard(await runNative(cliArgs));
         }
-        return Promise.resolve(failResult('Provide either ref or x+y coordinates'));
+        return failResult('Provide either ref or x+y coordinates');
     });
 }
 // Splits a chunk into segments where no segment, after space→%s encoding,

scripts/cdp-bridge/src/runners/keyboard-guard.ts

@@ -0,0 +1,44 @@
+export function resolveKeyboardGuard(env: NodeJS.ProcessEnv): boolean {
+  const raw = (env.RN_KEYBOARD_GUARD ?? '').trim().toLowerCase();
+  return !(raw === '0' || raw === 'false');
+}
+
+const GUARDED_VERBS = new Set(['tap', 'press', 'longPress']);
+
+export function withKeyboardGuard<T extends object>(
+  payload: T,
+  verb: string,
+  env: NodeJS.ProcessEnv,
+): T & { guardKeyboard?: boolean } {
+  if (!GUARDED_VERBS.has(verb)) return payload;
+  return { ...payload, guardKeyboard: resolveKeyboardGuard(env) };
+}
+
+interface ToolResultLike {
+  content: Array<{ type: 'text'; text: string }>;
+  isError?: true;
+}
+
+export function surfaceKeyboardGuard<T extends ToolResultLike>(result: T): T {
+  const text = result.content?.[0]?.text;
+  if (typeof text !== 'string') return result;
+
+  let envelope: Record<string, unknown>;
+  try {
+    envelope = JSON.parse(text) as Record<string, unknown>;
+  } catch {
+    return result;
+  }
+
+  const data = envelope.data as Record<string, unknown> | undefined;
+  const keyboardGuard = data?.keyboardGuard;
+  if (typeof keyboardGuard !== 'string') return result;
+
+  const meta = (envelope.meta as Record<string, unknown> | undefined) ?? {};
+  envelope.meta = { ...meta, keyboardGuard };
+
+  return {
+    ...result,
+    content: [{ type: 'text' as const, text: JSON.stringify(envelope) }],
+  };
+}

scripts/cdp-bridge/src/runners/rn-android-runner-client.ts

@@ -11,6 +11,7 @@ import { okResult, failResult } from '../utils.js';
 import { updateRefMapFromFlat, getCachedMetadata, type FlatNode } from '../fast-runner-ref-map.js';
 import { findFreePort } from './free-port.js';
 import { join } from 'node:path';
+import { withKeyboardGuard } from './keyboard-guard.js';
 
 const execFileAsync = promisify(execFile);
 
@@ -592,7 +593,9 @@ export async function runAndroid(args: RunAndroidArgs): Promise<ToolResult> {
   let resp: RunnerResponse;
   try {
     await startAndroidRunner(args.deviceId, args.bundleId);
-    resp = await postCommand(body);
+    resp = await postCommand(
+      withKeyboardGuard(body, args.command, process.env) as Record<string, unknown>,
+    );
   } catch (err) {
     const m = errMessage(err);
     // GH#243: a connection failure (runner just restarted after a flow, or can't bind

scripts/cdp-bridge/src/runners/rn-fast-runner-client.ts

@@ -7,6 +7,7 @@ import { okResult, failResult } from '../utils.js';
 import type { FastRunnerState } from '../types.js';
 import { updateRefMapFromFlat, getCachedMetadata, type FlatNode } from '../fast-runner-ref-map.js';
 import { isPortFree } from './free-port.js';
+import { withKeyboardGuard } from './keyboard-guard.js';
 
 const DEFAULT_PORT = 22088;
 const READY_TIMEOUT_MS = 30_000;
@@ -712,7 +713,9 @@ export async function runIOS(args: RunIOSArgs): Promise<ToolResult> {
   if (args.depth !== undefined) body.depth = args.depth;
   if (args.scope !== undefined) body.scope = args.scope;
 
-  const resp = await postCommand(body);
+  const resp = await postCommand(
+    withKeyboardGuard(body, args.command, process.env) as Record<string, unknown>,
+  );
   if (!resp.ok) {
     const message = resp.error?.message ?? 'runner returned !ok with no error';
     const code = resp.error?.code;

scripts/cdp-bridge/src/tools/device-interact.ts

@@ -16,6 +16,7 @@ import {
   stopFastRunner,
 } from '../runners/rn-fast-runner-client.js';
 import { stopAndroidRunner } from '../runners/rn-android-runner-client.js';
+import { surfaceKeyboardGuard } from '../runners/keyboard-guard.js';
 import { withSession } from '../utils.js';
 import type { ToolResult } from '../utils.js';
 import { okResult, failResult, createStepTimer } from '../utils.js';
@@ -251,7 +252,7 @@ export async function pressCandidate(
 ): Promise<ToolResult> {
   const ref = candidate.ref.startsWith('@') ? candidate.ref : `@${candidate.ref}`;
   if (action === 'click') {
-    return runNative(['press', ref]);
+    return surfaceKeyboardGuard(await runNative(['press', ref]));
   }
   return okResult({ ref: candidate.ref, label: candidate.label, testID: candidate.testID });
 }
@@ -460,7 +461,7 @@ export function createDevicePressHandler(): (args: PressArgs) => Promise<ToolRes
     if (args.doubleTap) cliArgs.push('--double-tap');
     if (args.count && args.count > 1) cliArgs.push('--count', String(args.count));
     if (args.holdMs && args.holdMs > 0) cliArgs.push('--hold-ms', String(args.holdMs));
-    const result = await runNative(cliArgs);
+    const result = surfaceKeyboardGuard(await runNative(cliArgs));
     if (!result.isError && args.waitForFocusMs && args.waitForFocusMs > 0) {
       await new Promise((r) => setTimeout(r, args.waitForFocusMs));
     }
@@ -478,18 +479,18 @@ interface LongPressArgs {
 }
 
 export function createDeviceLongPressHandler(): (args: LongPressArgs) => Promise<ToolResult> {
-  return withSession((args) => {
+  return withSession(async (args) => {
     if (args.ref) {
       const ref = args.ref.startsWith('@') ? args.ref : `@${args.ref}`;
       const cliArgs = ['press', ref, '--hold-ms', String(args.durationMs ?? 1000)];
-      return runNative(cliArgs);
+      return surfaceKeyboardGuard(await runNative(cliArgs));
     }
     if (args.x != null && args.y != null) {
       const cliArgs = ['longpress', String(args.x), String(args.y)];
       if (args.durationMs) cliArgs.push(String(args.durationMs));
-      return runNative(cliArgs);
+      return surfaceKeyboardGuard(await runNative(cliArgs));
     }
-    return Promise.resolve(failResult('Provide either ref or x+y coordinates'));
+    return failResult('Provide either ref or x+y coordinates');
   });
 }
 

scripts/cdp-bridge/test/unit/gh-370-keyboard-guard.test.js

@@ -0,0 +1,88 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  resolveKeyboardGuard,
+  withKeyboardGuard,
+  surfaceKeyboardGuard,
+} from '../../dist/runners/keyboard-guard.js';
+
+test('defaults ON when unset', () => assert.equal(resolveKeyboardGuard({}), true));
+
+test('OFF for 0/false (case/space-insensitive)', () => {
+  assert.equal(resolveKeyboardGuard({ RN_KEYBOARD_GUARD: '0' }), false);
+  assert.equal(resolveKeyboardGuard({ RN_KEYBOARD_GUARD: ' False ' }), false);
+});
+
+test('ON for any other value', () =>
+  assert.equal(resolveKeyboardGuard({ RN_KEYBOARD_GUARD: 'yes' }), true));
+
+test('withKeyboardGuard: tap/longPress only', () => {
+  assert.equal(withKeyboardGuard({ command: 'tap' }, 'tap', {}).guardKeyboard, true);
+  assert.equal(
+    withKeyboardGuard({ command: 'longPress' }, 'longPress', { RN_KEYBOARD_GUARD: '0' })
+      .guardKeyboard,
+    false,
+  );
+  assert.equal('guardKeyboard' in withKeyboardGuard({ command: 'swipe' }, 'swipe', {}), false);
+});
+
+function toolResult(envelope) {
+  return { content: [{ type: 'text', text: JSON.stringify(envelope) }] };
+}
+
+test('surfaceKeyboardGuard: iOS envelope maps data.keyboardGuard to meta.keyboardGuard', () => {
+  const result = toolResult({
+    ok: true,
+    data: {
+      message: 'tapped',
+      gestureStartUptimeMs: 100,
+      gestureEndUptimeMs: 140,
+      x: 10,
+      y: 20,
+      referenceWidth: 390,
+      referenceHeight: 844,
+      keyboardGuard: 'dismissed',
+    },
+  });
+  const mapped = surfaceKeyboardGuard(result);
+  const envelope = JSON.parse(mapped.content[0].text);
+  assert.equal(envelope.meta.keyboardGuard, 'dismissed');
+  assert.equal(envelope.data.keyboardGuard, 'dismissed');
+});
+
+test('surfaceKeyboardGuard: Android envelope maps data.keyboardGuard to meta.keyboardGuard', () => {
+  const result = toolResult({
+    ok: true,
+    data: { x: 10, y: 20, tapped: true, keyboardGuard: 'dismissed' },
+  });
+  const mapped = surfaceKeyboardGuard(result);
+  const envelope = JSON.parse(mapped.content[0].text);
+  assert.equal(envelope.meta.keyboardGuard, 'dismissed');
+});
+
+test('surfaceKeyboardGuard: preserves existing meta fields', () => {
+  const result = toolResult({
+    ok: true,
+    data: { keyboardGuard: 'not_occluded' },
+    meta: { recovered: 'agent-device-runner-leak' },
+  });
+  const mapped = surfaceKeyboardGuard(result);
+  const envelope = JSON.parse(mapped.content[0].text);
+  assert.equal(envelope.meta.keyboardGuard, 'not_occluded');
+  assert.equal(envelope.meta.recovered, 'agent-device-runner-leak');
+});
+
+test('surfaceKeyboardGuard: absent field leaves result unchanged (no meta.keyboardGuard key)', () => {
+  const result = toolResult({ ok: true, data: { message: 'tapped' } });
+  const mapped = surfaceKeyboardGuard(result);
+  assert.equal(mapped, result);
+  const envelope = JSON.parse(mapped.content[0].text);
+  assert.equal('meta' in envelope, false);
+});
+
+test('surfaceKeyboardGuard: non-JSON content is returned unchanged without throwing', () => {
+  const result = { content: [{ type: 'text', text: 'not json' }] };
+  assert.doesNotThrow(() => surfaceKeyboardGuard(result));
+  const mapped = surfaceKeyboardGuard(result);
+  assert.equal(mapped, result);
+});

scripts/rn-android-runner/app/build.gradle.kts

@@ -27,6 +27,7 @@ android {
 }
 
 dependencies {
+    testImplementation("junit:junit:4.13.2")
     androidTestImplementation("androidx.test:runner:1.6.2")
     androidTestImplementation("androidx.test:rules:1.6.1")
     androidTestImplementation("androidx.test.ext:junit:1.2.1")