fix: capture android helper window roots

Author: thymikee

android-snapshot-helper/README.md

@@ -1,7 +1,10 @@
 # Android Snapshot Helper
 
 Small instrumentation APK used to capture Android accessibility snapshots without relying on
-`uiautomator dump`'s fixed idle wait behavior.
+`uiautomator dump`'s fixed idle wait behavior. The helper enables Android's interactive-window
+retrieval flag and serializes every accessible window root returned by `UiAutomation.getWindows()`
+so keyboards and system overlays can appear in the same snapshot. If interactive window roots are
+unavailable, it falls back to the active-window root.
 
 The helper is intentionally provider-neutral. Local `adb`, cloud ADB tunnels, and remote device
 providers can all install and run the same APK as long as they can execute ADB-style operations.
@@ -50,6 +53,8 @@ The final instrumentation result includes:
 - `maxDepth`
 - `maxNodes`
 - `rootPresent`
+- `captureMode` (`interactive-windows` or `active-window`)
+- `windowCount`
 - `nodeCount`
 - `truncated`
 - `elapsedMs`

android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java

@@ -1,12 +1,15 @@
 package com.callstack.agentdevice.snapshothelper;
 
+import android.accessibilityservice.AccessibilityServiceInfo;
 import android.app.Instrumentation;
 import android.app.UiAutomation;
 import android.graphics.Rect;
 import android.os.Bundle;
 import android.util.Base64;
 import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityWindowInfo;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.TimeoutException;
 
@@ -52,6 +55,8 @@ public void onStart() {
       emitChunks(capture.xml);
       result.putString("ok", "true");
       result.putString("rootPresent", Boolean.toString(capture.rootPresent));
+      result.putString("captureMode", capture.captureMode);
+      result.putString("windowCount", Integer.toString(capture.windowCount));
       result.putString("nodeCount", Integer.toString(capture.nodeCount));
       result.putString("truncated", Boolean.toString(capture.truncated));
       result.putString("elapsedMs", Long.toString(System.currentTimeMillis() - startedAtMs));
@@ -70,6 +75,7 @@ private CaptureResult captureXml(
       long waitForIdleTimeoutMs, long timeoutMs, int maxDepth, int maxNodes)
       throws TimeoutException {
     UiAutomation automation = getUiAutomation();
+    enableInteractiveWindowRetrieval(automation);
     if (waitForIdleTimeoutMs > 0) {
       try {
         // Best-effort settle: avoids empty roots without inheriting UIAutomator's long idle wait.
@@ -79,19 +85,87 @@ private CaptureResult captureXml(
       }
     }
 
-    AccessibilityNodeInfo root = automation.getRootInActiveWindow();
     CaptureStats stats = new CaptureStats();
     StringBuilder xml = new StringBuilder();
     xml.append("<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>");
     xml.append("<hierarchy rotation=\"0\">");
-    if (root != null) {
-      appendNode(xml, root, 0, 0, maxDepth, maxNodes, stats);
+    int windowCount = appendInteractiveWindowRoots(xml, automation, maxDepth, maxNodes, stats);
+    String captureMode = "interactive-windows";
+    if (windowCount == 0) {
+      AccessibilityNodeInfo root = automation.getRootInActiveWindow();
+      try {
+        if (root != null) {
+          appendNode(xml, root, 0, 0, maxDepth, maxNodes, stats);
+          windowCount = 1;
+        }
+        captureMode = "active-window";
+      } finally {
+        if (root != null) {
+          root.recycle();
+        }
+      }
     }
     xml.append("</hierarchy>");
-    if (root != null) {
-      root.recycle();
+    return new CaptureResult(
+        xml.toString(), windowCount > 0, captureMode, windowCount, stats.nodeCount, stats.truncated);
+  }
+
+  private static void enableInteractiveWindowRetrieval(UiAutomation automation) {
+    AccessibilityServiceInfo serviceInfo;
+    try {
+      serviceInfo = automation.getServiceInfo();
+    } catch (RuntimeException error) {
+      return;
+    }
+    if (serviceInfo == null) {
+      return;
+    }
+    if ((serviceInfo.flags & AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS) != 0) {
+      return;
+    }
+    serviceInfo.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS;
+    try {
+      automation.setServiceInfo(serviceInfo);
+    } catch (RuntimeException ignored) {
+      // Fall back to active-window capture if the platform rejects dynamic service flags.
+    }
+  }
+
+  private static int appendInteractiveWindowRoots(
+      StringBuilder xml,
+      UiAutomation automation,
+      int maxDepth,
+      int maxNodes,
+      CaptureStats stats) {
+    List<AccessibilityWindowInfo> windows;
+    try {
+      windows = automation.getWindows();
+    } catch (RuntimeException error) {
+      return 0;
+    }
+    int windowCount = 0;
+    for (int index = 0; index < windows.size(); index += 1) {
+      if (stats.nodeCount >= maxNodes) {
+        stats.truncated = true;
+        break;
+      }
+      AccessibilityWindowInfo window = windows.get(index);
+      AccessibilityNodeInfo root = null;
+      try {
+        root = window.getRoot();
+        if (root == null) {
+          continue;
+        }
+        appendNode(xml, root, windowCount, 0, maxDepth, maxNodes, stats);
+        windowCount += 1;
+      } finally {
+        if (root != null) {
+          root.recycle();
+        }
+        window.recycle();
+      }
     }
-    return new CaptureResult(xml.toString(), root != null, stats.nodeCount, stats.truncated);
+    return windowCount;
   }
 
   private void emitChunks(String payload) {
@@ -262,12 +336,22 @@ private static final class CaptureStats {
   private static final class CaptureResult {
     final String xml;
     final boolean rootPresent;
+    final String captureMode;
+    final int windowCount;
     final int nodeCount;
     final boolean truncated;
 
-    CaptureResult(String xml, boolean rootPresent, int nodeCount, boolean truncated) {
+    CaptureResult(
+        String xml,
+        boolean rootPresent,
+        String captureMode,
+        int windowCount,
+        int nodeCount,
+        boolean truncated) {
       this.xml = xml;
       this.rootPresent = rootPresent;
+      this.captureMode = captureMode;
+      this.windowCount = windowCount;
       this.nodeCount = nodeCount;
       this.truncated = truncated;
     }

src/platforms/android/__tests__/snapshot-helper.test.ts

@@ -45,6 +45,8 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', ()
       maxDepth: '128',
       maxNodes: '5000',
       rootPresent: 'true',
+      captureMode: 'interactive-windows',
+      windowCount: '2',
       nodeCount: '1',
       truncated: 'false',
       elapsedMs: '42',
@@ -62,6 +64,8 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', ()
     maxDepth: 128,
     maxNodes: 5000,
     rootPresent: true,
+    captureMode: 'interactive-windows',
+    windowCount: 2,
     nodeCount: 1,
     truncated: false,
     elapsedMs: 42,
@@ -95,12 +99,14 @@ test('parseAndroidSnapshotHelperOutput decodes UTF-8 across byte chunk boundarie
 test('parseAndroidSnapshotHelperSnapshot returns shaped nodes', () => {
   const output = helperOutput({
     chunks: [
-      '<hierarchy><node text="Continue" class="android.widget.Button" bounds="[1,2][21,42]" clickable="true" /></hierarchy>',
+      '<hierarchy><node text="Continue" class="android.widget.Button" bounds="[1,2][21,42]" clickable="true" /><node text="Keyboard suggestion" class="android.widget.TextView" bounds="[1,44][121,84]" /></hierarchy>',
     ],
     result: {
       ok: 'true',
       outputFormat: 'uiautomator-xml',
-      nodeCount: '1',
+      captureMode: 'interactive-windows',
+      windowCount: '2',
+      nodeCount: '2',
     },
   });
 
@@ -109,7 +115,10 @@ test('parseAndroidSnapshotHelperSnapshot returns shaped nodes', () => {
   assert.equal(parsed.nodes[0]?.label, 'Continue');
   assert.equal(parsed.nodes[0]?.hittable, true);
   assert.deepEqual(parsed.nodes[0]?.rect, { x: 1, y: 2, width: 20, height: 40 });
-  assert.equal(parsed.metadata.nodeCount, 1);
+  assert.equal(parsed.nodes[1]?.label, 'Keyboard suggestion');
+  assert.equal(parsed.metadata.captureMode, 'interactive-windows');
+  assert.equal(parsed.metadata.windowCount, 2);
+  assert.equal(parsed.metadata.nodeCount, 2);
 });
 
 test('parseAndroidSnapshotHelperXml returns shaped nodes from captured helper output', () => {

src/platforms/android/__tests__/snapshot.test.ts

@@ -159,6 +159,8 @@ function helperOutput(xml: string): string {
     'INSTRUMENTATION_RESULT: maxDepth=128',
     'INSTRUMENTATION_RESULT: maxNodes=5000',
     'INSTRUMENTATION_RESULT: rootPresent=true',
+    'INSTRUMENTATION_RESULT: captureMode=interactive-windows',
+    'INSTRUMENTATION_RESULT: windowCount=1',
     'INSTRUMENTATION_RESULT: nodeCount=1',
     'INSTRUMENTATION_RESULT: truncated=false',
     'INSTRUMENTATION_RESULT: elapsedMs=12',
@@ -250,6 +252,8 @@ test('snapshotAndroid uses configured helper before stock uiautomator', async ()
   assert.equal(result.androidSnapshot.backend, 'android-helper');
   assert.equal(result.androidSnapshot.helperVersion, '0.13.3');
   assert.equal(result.androidSnapshot.installReason, 'current');
+  assert.equal(result.androidSnapshot.captureMode, 'interactive-windows');
+  assert.equal(result.androidSnapshot.windowCount, 1);
   assert.deepEqual(timeouts, [30000, 13000]);
   assert.equal(mockRunCmd.mock.calls.length, 0);
 });

src/platforms/android/snapshot-helper.ts

@@ -88,6 +88,8 @@ export type AndroidSnapshotHelperMetadata = {
   maxDepth?: number;
   maxNodes?: number;
   rootPresent?: boolean;
+  captureMode?: 'interactive-windows' | 'active-window';
+  windowCount?: number;
   nodeCount?: number;
   truncated?: boolean;
   elapsedMs?: number;
@@ -439,12 +441,20 @@ function readHelperMetadata(finalResult: Record<string, string>): AndroidSnapsho
     maxDepth: readOptionalNumber(finalResult.maxDepth),
     maxNodes: readOptionalNumber(finalResult.maxNodes),
     rootPresent: readOptionalBoolean(finalResult.rootPresent),
+    captureMode: readOptionalCaptureMode(finalResult.captureMode),
+    windowCount: readOptionalNumber(finalResult.windowCount),
     nodeCount: readOptionalNumber(finalResult.nodeCount),
     truncated: readOptionalBoolean(finalResult.truncated),
     elapsedMs: readOptionalNumber(finalResult.elapsedMs),
   };
 }
 
+function readOptionalCaptureMode(
+  value: string | undefined,
+): AndroidSnapshotHelperMetadata['captureMode'] {
+  return value === 'interactive-windows' || value === 'active-window' ? value : undefined;
+}
+
 export function parseAndroidSnapshotHelperSnapshot(
   output: string,
   options: SnapshotOptions = {},

src/platforms/android/snapshot-types.ts

@@ -11,6 +11,8 @@ export type AndroidSnapshotBackendMetadata = {
   maxDepth?: number;
   maxNodes?: number;
   rootPresent?: boolean;
+  captureMode?: 'interactive-windows' | 'active-window';
+  windowCount?: number;
   nodeCount?: number;
   helperTruncated?: boolean;
   elapsedMs?: number;

src/platforms/android/snapshot.ts

@@ -117,6 +117,8 @@ async function captureAndroidUiHierarchy(
           maxDepth: capture.metadata.maxDepth,
           maxNodes: capture.metadata.maxNodes,
           rootPresent: capture.metadata.rootPresent,
+          captureMode: capture.metadata.captureMode,
+          windowCount: capture.metadata.windowCount,
           nodeCount: capture.metadata.nodeCount,
           helperTruncated: capture.metadata.truncated,
           elapsedMs: capture.metadata.elapsedMs,

website/docs/docs/client-api.md

@@ -130,6 +130,10 @@ const output = await captureAndroidSnapshotWithHelper({
 const snapshot = parseAndroidSnapshotHelperXml(output.xml, output.metadata);
 ```
 
+Helper captures report `metadata.captureMode` as `interactive-windows` when Android returns
+interactive window roots, or `active-window` when the helper falls back to
+`getRootInActiveWindow()`. `metadata.windowCount` is the number of serialized roots.
+
 ## Command methods
 
 Use `client.command.<method>()` for command-level device actions. It uses the same daemon transport path as the higher-level client methods, including session metadata, tenant/run/lease fields, normalized daemon errors, and remote artifact handling.

website/docs/docs/commands.md

@@ -199,7 +199,9 @@ agent-device get attrs @e1
 - Android snapshots use stock UIAutomator by default. To try the released Android snapshot helper
   before stock UIAutomator, set both `AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_APK` and
   `AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_MANIFEST`; helper failures fall back to stock UIAutomator
-  and include `androidSnapshot.fallbackReason` in typed results.
+  and include `androidSnapshot.fallbackReason` in typed results. The helper serializes Android
+  interactive window roots when available, so keyboard and system-overlay nodes can appear alongside
+  the app root; `androidSnapshot.captureMode` and `androidSnapshot.windowCount` describe the capture.
 - `diff snapshot` compares the current snapshot with the previous session baseline and then updates baseline.
 - `snapshot --diff` is an alias for `diff snapshot`.