From 2438e7e4c3a7253bae721a2bbfb15b0964c3b9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 31 May 2026 12:41:54 +0200 Subject: [PATCH 1/4] perf(ios): early-exit text-entry readiness when the keyboard is visible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The XCUITest text-entry focus/readiness loops keyed their fast-exit on focusedTextInput(), which is intentionally hardcoded to return nil on iOS (focus predicates are stale there). As a result stabilizeTextInputBeforeTyping always burned its full focusTimeout (0.4s) and waitForTextEntryReadiness burned its full readinessTimeout (2.0s) in the normal case where the software keyboard appears — ~2.4s of dead wait before a single keystroke on every type/fill. The software keyboard becoming visible is the reliable iOS readiness signal, so both loops now return as soon as isKeyboardVisible() is true. The warmup-first-char echo check and post-type verify/repair remain as drop safety nets. Measured on iPhone 17 sim (Settings search field), median type time: 25 chars: 3342ms -> 1379ms (2.4x) 52 chars: 3969ms -> 2190ms 313 chars: 10.3s -> 8.6s (remainder is genuine per-char XCUITest typing) Reliability unchanged: 64/65 trials exact (incl. a 50-word lorem ipsum, verified by read-back + screenshot); the lone miss triggered the existing verify/repair. --- .../RunnerTests+Interaction.swift | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index cd231a953..cdc402ab2 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -416,6 +416,12 @@ extension RunnerTests { if let focused = focusedTextInput(app: app) { return focused } + // focusedTextInput is intentionally nil on iOS, so the software keyboard becoming + // visible is the reliable readiness signal. Exit as soon as it is up instead of + // always burning the full focusTimeout. + if isKeyboardVisible(app: app) { + return latest + } sleepFor(TextEntryTiming.pollInterval) } return latest @@ -878,6 +884,13 @@ extension RunnerTests { return focused } } + // The software keyboard being visible is the reliable readiness signal on iOS + // (focusedTextInput is intentionally nil there). Once it is up the field is accepting + // input, so return immediately rather than burning the full readinessTimeout — the + // warmup-first-char echo check and post-type verify/repair remain as drop safety nets. + if isKeyboardVisible(app: app) { + return latest + } sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app) if !sawSoftwareKeyboard && Date() >= hardwareKeyboardFallback && latest != nil { return latest From d0d2cc3f64c578c2787b14612957e06d48a15c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 31 May 2026 13:00:31 +0200 Subject: [PATCH 2/4] fix(ios): don't clear an already-empty text field (fixes fill mis-navigation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clearTextInput unconditionally ran moveCaretToEnd (an edge-tap computed from the element frame) + a 24-key delete burst, even when the field was empty. On a field that repositions on focus — e.g. the Settings search bar jumping bottom->top and revealing a 'Suggestions' list — that edge-tap used a stale frame and landed on an adjacent row (Developer), navigating away instead of clearing. fill (replace) into the search field went to the Developer pane (0/3 correct). Skip the clear entirely when the field's value is already empty (placeholder treated as empty): replacing into an empty field is a no-op, and skipping avoids the stray edge-tap. fill into the Settings search now types correctly and stays put: 5/5 exact (read-back + screenshot). --- .../RunnerTests+Interaction.swift | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index cdc402ab2..12c7d7bb7 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -316,6 +316,15 @@ extension RunnerTests { } func clearTextInput(_ element: XCUIElement) { + // Nothing to clear: skip both the delete burst and the moveCaretToEnd edge-tap. The + // edge-tap computes a point from the element frame, which can be stale after the field + // repositions on focus (e.g. the Settings search bar jumps bottom->top and reveals a + // "Suggestions" list) — tapping there navigates away instead of clearing. Replacing into + // an already-empty field is a no-op, so returning early is also semantically correct. + let existing = editableTextValue(for: element, treatingPlaceholderAsEmpty: true) ?? "" + if existing.isEmpty { + return + } #if !os(tvOS) moveCaretToEnd(element: element) #endif From 943ab0aa4ee2263befae26321d706fdbebf657fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 31 May 2026 13:38:43 +0200 Subject: [PATCH 3/4] perf(ios): resolve text fields via typed queries, not full-tree enumeration textInputAt used app.descendants(.any).allElementsBoundByIndex (snapshots EVERY element) to find the text input at a point. fill drove this repeatedly: once it has coordinates, resolveTextEntryElement re-runs textInputAt on every verify/repair poll iteration whenever the focused-field reference goes stale (e.g. the Settings search bar repositioning bottom->top), so the full-tree enum dominated fill latency. Query the text-input element types directly (app.textFields/secureTextFields/ searchFields/textViews) instead. Same matches, but XCUITest resolves typed queries without snapshotting the whole tree. Measured (iPhone 17 sim, warm runner): fill 25 chars ~14.5s -> ~4.5s (3.2x), 6/6 exact. Same primitive #632 killed for get text. --- .../RunnerTests+Interaction.swift | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 12c7d7bb7..a331e201d 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -337,17 +337,22 @@ extension RunnerTests { let point = CGPoint(x: x, y: y) var matched: XCUIElement? let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + // Query the text-input element types directly instead of enumerating the entire tree + // (app.descendants(.any).allElementsBoundByIndex snapshots every element and is ~10x + // slower — it dominated fill latency because resolveTextEntryElement re-runs this on + // each verify/repair poll once the focused field reference goes stale). // Prefer the smallest matching field so nested editable controls win over large containers. - let candidates = app.descendants(matching: .any).allElementsBoundByIndex + let candidates = [ + app.textFields, + app.secureTextFields, + app.searchFields, + app.textViews, + ] + .flatMap { $0.allElementsBoundByIndex } .filter { element in guard element.exists else { return false } - switch element.elementType { - case .textField, .secureTextField, .searchField, .textView: - let frame = element.frame - return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2) - default: - return false - } + let frame = element.frame + return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2) } .sorted { left, right in let leftArea = max(1, left.frame.width * left.frame.height) From 296364fcf1cb6996a179a9d6f51760ed8d9bed33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 31 May 2026 14:20:44 +0200 Subject: [PATCH 4/4] =?UTF-8?q?fix(ios):=20address=20review=20=E2=80=94=20?= =?UTF-8?q?focus-change=20gate=20+=20secure-field=20clear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review P1 (focus race): the isKeyboardVisible early-exit in stabilizeTextInputBeforeTyping and waitForTextEntryReadiness fired the instant the keyboard was visible — but when it was ALREADY up from a previous field (back-to-back fills), that is before first-responder moves to the newly-tapped field, so app.typeText could target the old field. Gate the fast-path on a keyboard hidden->visible TRANSITION via a shared keyboardBecameVisible(wasVisibleAtEntry:) helper; when the keyboard was already up, fall back to the settle/timeout (the prior, correct behavior) instead of the ~2.4s dead wait the fresh case avoids. Review P1 (F2): clearTextInput used editableTextValue(...) ?? "" and skipped clearing on empty — but editableTextValue returns nil for secure (and unknown) fields, so secure fields were NEVER cleared and replace concatenated stale+new. Distinguish nil (clear) from "" (skip). Device-validated: fresh fill fast-path preserved + exact; a second fill with the keyboard already up still types into the correct field and replaces (not concatenates). --- .../RunnerTests+Interaction.swift | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index a331e201d..f28cacc14 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -316,13 +316,16 @@ extension RunnerTests { } func clearTextInput(_ element: XCUIElement) { - // Nothing to clear: skip both the delete burst and the moveCaretToEnd edge-tap. The - // edge-tap computes a point from the element frame, which can be stale after the field - // repositions on focus (e.g. the Settings search bar jumps bottom->top and reveals a - // "Suggestions" list) — tapping there navigates away instead of clearing. Replacing into - // an already-empty field is a no-op, so returning early is also semantically correct. - let existing = editableTextValue(for: element, treatingPlaceholderAsEmpty: true) ?? "" - if existing.isEmpty { + // Skip the clear (delete burst + moveCaretToEnd edge-tap) ONLY when we can confirm the + // field is empty. Why skip: the edge-tap computes a point from the element frame, which can + // be stale after the field repositions on focus (e.g. the Settings search bar jumps + // bottom->top and reveals a "Suggestions" list) — tapping there navigates away instead of + // clearing; and replacing into an already-empty field is a no-op anyway. + // editableTextValue returns nil for secure (and unknown) fields, where we CANNOT confirm + // emptiness — those must still be cleared, or replace would concatenate stale + new text. + // So distinguish nil (clear) from "" (skip). + if let existing = editableTextValue(for: element, treatingPlaceholderAsEmpty: true), + existing.isEmpty { return } #if !os(tvOS) @@ -425,15 +428,15 @@ extension RunnerTests { return target #else let latest = target + let keyboardVisibleAtEntry = isKeyboardVisible(app: app) let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout) while Date() < deadline { if let focused = focusedTextInput(app: app) { return focused } - // focusedTextInput is intentionally nil on iOS, so the software keyboard becoming - // visible is the reliable readiness signal. Exit as soon as it is up instead of - // always burning the full focusTimeout. - if isKeyboardVisible(app: app) { + // focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to + // visible after our tap as the focus-moved signal. Don't fast-path when it was already up. + if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) { return latest } sleepFor(TextEntryTiming.pollInterval) @@ -886,6 +889,7 @@ extension RunnerTests { ) -> XCUIElement? { #if os(iOS) var latest = resolveTextEntryElement(app: app, target: target) + let keyboardVisibleAtEntry = isKeyboardVisible(app: app) let deadline = Date().addingTimeInterval(timeout) let hardwareKeyboardFallback = Date().addingTimeInterval( min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout) @@ -898,11 +902,12 @@ extension RunnerTests { return focused } } - // The software keyboard being visible is the reliable readiness signal on iOS - // (focusedTextInput is intentionally nil there). Once it is up the field is accepting - // input, so return immediately rather than burning the full readinessTimeout — the - // warmup-first-char echo check and post-type verify/repair remain as drop safety nets. - if isKeyboardVisible(app: app) { + // Fast-path on a keyboard hidden->visible transition: our tapped field gained focus, so + // return immediately instead of burning the full readinessTimeout (warmup-first-char echo + // + post-type verify/repair remain as drop safety nets). When the keyboard was ALREADY up + // (back-to-back fills), this isn't a focus signal — fall through to the settle/timeout so + // text isn't sent to the previously-focused field. + if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) { return latest } sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app) @@ -961,6 +966,15 @@ extension RunnerTests { return visibleKeyboardFrame(app: app) != nil } + /// A focus-moved signal for iOS text entry, where `focusedTextInput` is intentionally nil. + /// The software keyboard TRANSITIONING from hidden (at entry) to visible means the field we + /// just tapped gained first-responder. If the keyboard was ALREADY up (e.g. back-to-back + /// fills into different fields), its visibility is not evidence focus moved to the new field, + /// so callers must keep waiting rather than typing into the previously-focused field. + private func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool { + return !wasVisibleAtEntry && isKeyboardVisible(app: app) + } + private func keyboardElementExists(app: XCUIApplication) -> Bool { #if os(iOS) var exists = false