fix: make fill clear input and progressively fill the input (#17)

* fix: make fill clear input and progressively fill the input

* fixup ios

* improve text fill reliability

Author: thymikee

AGENTS.md

@@ -26,19 +26,16 @@ Instructions for AI coding agents working with this codebase.
 2) Daemon (`src/daemon.ts`) resolves device, tracks session state, and calls `dispatchCommand`.
 3) `dispatchCommand` selects platform interactor.
 4) iOS simulator path:
-   - Prefer `simctl` input when available (`simctlSupportsInput`).
-   - Snapshot default backend: AX (`snapshotAx`), fallback to iOS runner if AX is unavailable.
-   - Fallback to iOS runner via `runIosRunnerCommand` for tap/type/swipe/list.
+   - Snapshot default backend: XCTest for everything, fallback to AX if XCTest returns 0 nodes.
 5) iOS runner uses xcodebuild `test-without-building` with an injected `.xctestrun`,
    starts an `NWListener` HTTP server inside the UI test bundle, and executes UI actions.
 
 ## Key commands (local)
 
-- Build iOS runner: `xcodebuild build-for-testing -project ios-runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj -scheme AgentDeviceRunner -destination "platform=iOS Simulator,id=<UDID>" -derivedDataPath ~/.agent-device/ios-runner/derived`
-- Build AX snapshot tool: `swift build -c release` in `ios-runner/AXSnapshot`
+- Build iOS runner: `pnpm build:xcuitest`
+- Build AX snapshot tool: `pnpm build:axsnapshot`
 - Build everything: `pnpm build:all`
-- Run command: `node bin/agent-device.mjs --platform ios --udid <UDID> open settings --verbose`
-- Scroll: `node bin/agent-device.mjs --platform ios --udid <UDID> scroll down 0.5 --verbose`
+- Run command: `pnpm ad`
 
 ## Run and Test
 

README.md

@@ -121,6 +121,13 @@ Find (semantic):
 - `find text|label|value|role|id <value> <action> [value]` for specific locators.
 - Actions: `click` (default), `fill`, `type`, `focus`, `get text`, `get attrs`, `wait [timeout]`, `exists`.
 
+Android fill reliability:
+- `fill` clears the current value, then enters text.
+- `type` enters text into the focused field without clearing.
+- `fill` now verifies the entered value on Android.
+- If value does not match, agent-device clears the field and retries once with slower typing.
+- This reduces IME-related character swaps on long strings (e.g. emails and IDs).
+
 Settings helpers (simulators):
 - `settings wifi on|off`
 - `settings airplane on|off`

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -81,9 +81,7 @@ final class RunnerTests: XCTestCase {
       return
     }
     NSLog("AGENT_DEVICE_RUNNER_WAITING")
-    let timeout = resolveRunnerTimeout()
-    let effectiveTimeout = timeout > 0 ? timeout : 24 * 60 * 60
-    let result = XCTWaiter.wait(for: [expectation], timeout: effectiveTimeout)
+    let result = XCTWaiter.wait(for: [expectation], timeout: 24 * 60 * 60)
     NSLog("AGENT_DEVICE_RUNNER_WAIT_RESULT=%@", String(describing: result))
     if result != .completed {
       XCTFail("runner wait ended with \(result)")
@@ -238,7 +236,19 @@ final class RunnerTests: XCTestCase {
       guard let text = command.text else {
         return Response(ok: false, error: ErrorPayload(message: "type requires text"))
       }
-      activeApp.typeText(text)
+      if command.clearFirst == true {
+        guard let focused = focusedTextInput(app: activeApp) else {
+          return Response(ok: false, error: ErrorPayload(message: "no focused text input to clear"))
+        }
+        clearTextInput(focused)
+        focused.typeText(text)
+        return Response(ok: true, data: DataPayload(message: "typed"))
+      }
+      if let focused = focusedTextInput(app: activeApp) {
+        focused.typeText(text)
+      } else {
+        activeApp.typeText(text)
+      }
       return Response(ok: true, data: DataPayload(message: "typed"))
     case .swipe:
       guard let direction = command.direction else {
@@ -343,6 +353,48 @@ final class RunnerTests: XCTestCase {
     return element.exists ? element : nil
   }
 
+  private func clearTextInput(_ element: XCUIElement) {
+    moveCaretToEnd(element: element)
+    let count = estimatedDeleteCount(for: element)
+    let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
+    element.typeText(deletes)
+  }
+
+  private func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
+    let focused = app
+      .descendants(matching: .any)
+      .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+      .firstMatch
+    guard focused.exists else { return nil }
+
+    switch focused.elementType {
+    case .textField, .secureTextField, .searchField, .textView:
+      return focused
+    default:
+      return nil
+    }
+  }
+
+  private func moveCaretToEnd(element: XCUIElement) {
+    let frame = element.frame
+    guard !frame.isEmpty else {
+      element.tap()
+      return
+    }
+    let origin = element.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+    let target = origin.withOffset(
+      CGVector(dx: max(2, frame.width - 4), dy: max(2, frame.height / 2))
+    )
+    target.tap()
+  }
+
+  private func estimatedDeleteCount(for element: XCUIElement) -> Int {
+    let valueText = String(describing: element.value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    let base = valueText.isEmpty ? 24 : (valueText.count + 8)
+    return max(24, min(120, base))
+  }
+
   private func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
     let predicate = NSPredicate(
       format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
@@ -738,14 +790,6 @@ private func resolveRunnerPort() -> UInt16 {
   return 0
 }
 
-private func resolveRunnerTimeout() -> TimeInterval {
-  if let env = ProcessInfo.processInfo.environment["AGENT_DEVICE_RUNNER_TIMEOUT"],
-     let parsed = Double(env) {
-    return parsed
-  }
-  return 0
-}
-
 enum CommandType: String, Codable {
   case tap
   case type
@@ -772,6 +816,7 @@ struct Command: Codable {
   let command: CommandType
   let appBundleId: String?
   let text: String?
+  let clearFirst: Bool?
   let action: String?
   let x: Double?
   let y: Double?

skills/agent-device/SKILL.md

@@ -97,8 +97,8 @@ agent-device apps --metadata --platform android
 ```bash
 agent-device click @e1
 agent-device focus @e2
-agent-device fill @e2 "text"           # Tap then type
-agent-device type "text"               # Type into focused field
+agent-device fill @e2 "text"           # Clear then type (Android: verifies value and retries once on mismatch)
+agent-device type "text"               # Type into focused field without clearing
 agent-device press 300 500             # Tap by coordinates
 agent-device long-press 300 500 800    # Long press (where supported)
 agent-device scroll down 0.5
@@ -150,6 +150,9 @@ agent-device apps --platform android --user-installed
 - If AX returns the Simulator window or empty tree, restart Simulator or use `--backend xctest`.
 - Use `--session <name>` for parallel sessions; avoid device contention.
 - Use `--activity <component>` on Android to launch a specific activity (e.g. TV apps with LEANBACK).
+- Use `fill` when you want clear-then-type semantics.
+- Use `type` when you want to append/enter text without clearing.
+- On Android, prefer `fill` for important fields; it verifies entered text and retries once when IME reorders characters.
 
 ## References
 

src/core/dispatch.ts

@@ -174,7 +174,7 @@ export async function dispatchCommand(
         );
         await runIosRunnerCommand(
           device,
-          { command: 'type', text, appBundleId: context?.appBundleId },
+          { command: 'type', text, clearFirst: true, appBundleId: context?.appBundleId },
           { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath },
         );
       } else {

src/daemon.ts

@@ -1036,32 +1036,17 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
       if (!node?.rect) {
         return { ok: false, error: { code: 'COMMAND_FAILED', message: `Ref ${req.positionals[0]} not found or has no bounds` } };
       }
-      const refLabel = resolveRefLabel(node, session.snapshot.nodes);
-      const label = node.label?.trim();
-      if (session.device.platform === 'ios' && session.device.kind === 'simulator' && isTextInputType(node.type)) {
-        const coords = node.rect ? centerOfRect(node.rect) : null;
-        if (!coords) {
-          return {
-            ok: false,
-            error: { code: 'COMMAND_FAILED', message: `Ref ${req.positionals[0]} not found or has no bounds` },
-          };
-        }
-        await dispatchCommand(session.device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, {
-          ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
-        });
-        await runIosRunnerCommand(
-          session.device,
-          { command: 'type', text, appBundleId: session.appBundleId },
-          { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
-        );
-        recordAction(session, {
-          command,
-          positionals: req.positionals ?? [],
-          flags: req.flags ?? {},
-          result: { ref, refLabel: refLabel ?? label, action: 'fill', text },
-        });
-        return { ok: true, data: { ref } };
+      const nodeType = node.type ?? '';
+      if (nodeType && !isFillableType(nodeType, session.device.platform)) {
+        return {
+          ok: false,
+          error: {
+            code: 'INVALID_ARGS',
+            message: `fill requires a text input element, got "${nodeType}" for ${req.positionals[0]}. Select a text input ref or use click/focus + type.`,
+          },
+        };
       }
+      const refLabel = resolveRefLabel(node, session.snapshot.nodes);
       const { x, y } = centerOfRect(node.rect);
       const data = await dispatchCommand(
         session.device,
@@ -1649,16 +1634,6 @@ function isLabelUnique(nodes: SnapshotState['nodes'], label: string): boolean {
   return count === 1;
 }
 
-function isTextInputType(type: string | undefined): boolean {
-  const normalized = normalizeType(type ?? '');
-  return (
-    normalized === 'textfield' ||
-    normalized === 'textview' ||
-    normalized === 'searchfield' ||
-    normalized === 'textarea'
-  );
-}
-
 function pruneGroupNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] {
   const skippedDepths: number[] = [];
   const result: RawSnapshotNode[] = [];
@@ -1690,6 +1665,25 @@ function normalizeType(type: string): string {
   return value;
 }
 
+function isFillableType(type: string, platform: 'ios' | 'android'): boolean {
+  const normalized = normalizeType(type);
+  if (!normalized) return true;
+  if (platform === 'android') {
+    return (
+      normalized.includes('edittext') ||
+      normalized.includes('autocompletetextview')
+    );
+  }
+  return (
+    normalized.includes('textfield') ||
+    normalized.includes('securetextfield') ||
+    normalized.includes('searchfield') ||
+    normalized.includes('textview') ||
+    normalized.includes('textarea') ||
+    normalized === 'search'
+  );
+}
+
 function findNearestHittableAncestor(
   nodes: SnapshotState['nodes'],
   node: SnapshotState['nodes'][number],