feat(ui-automation): Add rs/1 runtime automation parity (#416)

Author: cameroncooke

.gitignore

@@ -117,3 +117,4 @@ DerivedData
 /.pr-learning
 /repros
 /.xcodebuildmcp
+/out.nosync

AGENTS.md

@@ -70,6 +70,7 @@ ESM TypeScript project (`type: module`). Key layers:
 
 ## Tool Development
 - Tool manifests in `manifests/tools/*.yaml` define `id`, `module`, `names.mcp` (snake_case), optional `names.cli` (kebab-case), predicates, and annotations
+- MCP `readOnlyHint` describes whether a tool mutates host/project state such as files, build artifacts, configuration, or external services. Simulator HID/UI actions that only tap, type, press, or gesture inside the simulator may remain `readOnlyHint: true`; do not flip them to `false` merely because app UI state changes.
 - Workflow manifests in `manifests/workflows/*.yaml` group tools and define exposure rules
 - Tool modules export a Zod `schema`, a pure `*Logic` function, and a `handler` built with `createTypedTool` or `createSessionAwareTool`
 - Resource modules export a `handler` (and a pure `*Logic` function); `uri`, `name`, `description`, and `mimeType` are declared in `manifests/resources/*.yaml`
@@ -95,6 +96,12 @@ When reading issues:
 - Use shared lock and atomic-write helpers for mutable shared files.
 - Prefer one-record-per-file registries over shared aggregate files.
 - Cleanup must verify ownership before deleting shared artifacts.
+- Multi-process safety means concurrent processes must not corrupt or delete each other's state.
+  It does not mean ephemeral runtime handles should become portable between invocation surfaces.
+- Keep runtime/session-scoped handles isolated unless the product explicitly defines a cross-process
+  contract. For example, UI automation `elementRef` values from runtime snapshots are handles for
+  the runtime/session that produced them, not durable IDs to share between separate MCP and CLI
+  invocations.
 - User-facing artifact/log paths in final text or structured output must use `displayPath()` from `src/utils/build-preflight.ts`, so paths are cwd-relative when possible or `~/...` instead of absolute home paths. Keep stored files at their real absolute paths; only normalize response/display values.
 
 ## Style

CHANGELOG.md

@@ -5,6 +5,69 @@
 ### Added
 
 - Added `nextSteps` hint lines to MCP `structuredContent` and CLI `--output json` envelopes so agents can consume follow-up actions without scraping text. CLI JSON renders shell command lines; MCP structured content renders MCP tool-call hints. Structured result schemas that include `nextSteps` now use schema version 2; existing version 1 schema files remain available for current validators.
+- Added `snapshot_ui sinceScreenHash` / CLI `--since-screen-hash` so callers can skip full runtime snapshot output when the screen hash is unchanged.
+- Added `batch` for executing multiple AXe UI automation steps in one simulator session.
+- Added `wait_for_ui` for polling runtime UI snapshots until UI predicates such as existence, enabled state, focus, text, or settled layout are satisfied. `textContains` can also wait on visible text without a selector when the match is unique.
+- Added structured element-ref `batch` tap steps, preserved same-screen refs after successful `tap` and `batch` actions, and improved UI automation guidance and next steps for one-observation interactions.
+- Added a `replaceExisting` option to `type_text` so agents can replace an existing text-field value instead of accidentally appending to it.
+- Added `drag` for element-ref based drag gestures, enabling agents to expand foreground sheets and drag real scroll/list regions without raw coordinate guesses.
+
+### Changed
+
+- Successful mutating UI automation calls now always attempt to refresh the runtime snapshot after the action instead of preserving or patching cached switch state.
+- Runtime snapshot guidance no longer advertises synthetic sheet swipe targets for foreground sheets. Agents should use real sheet grabber expansion and real descendant scroll/list targets with `drag` instead of inferred app/window-root sheet swipes.
+
+### Fixed
+
+- Fixed simulator launch failures before simulator-name resolution so they are not reported as macOS launch failures.
+- Fixed CLI JSON output so simulator-name resolution failures return the structured error envelope instead of plain stderr.
+- Fixed accessibility hierarchy tips so UI automation guidance prefers runtime element refs over raw coordinate guessing.
+- Fixed `swipe` distance handling so distance is a normalized stroke fraction used for endpoint calculation, and improved sheet/list scroll guidance so real descendant scroll containers are preferred over application/window root fallbacks.
+- Fixed compact runtime snapshots so top-level app and window refs are not advertised as swipe targets just because a generic descendant overflows their frame.
+- Fixed `wait_for_ui` focus waits so elements that do not expose focus state return a typed recoverable error instead of timing out.
+- Fixed invalid `touch` calls so structured output no longer reports a fake touch event when neither `down` nor `up` was requested.
+- Fixed compact runtime snapshots so standalone `other` elements, such as keyboard suggestions, are not advertised as swipe targets unless they behave like scrollable containers.
+- Fixed runtime snapshots so off-screen elements, and clipped elements whose activation point is offscreen, are not advertised as actionable targets.
+- Fixed full-screen swipe gestures so app-level scroll refs avoid unsafe screen edges such as the status bar and notch area.
+- Clarified runtime snapshot tips so agents know element refs are snapshot-specific and must come from the latest `snapshot_ui` or `wait_for_ui` output, and only show swipe guidance when the snapshot includes a scroll ref.
+- Made `wait_for_ui` `textContains` matching case-insensitive so assertions survive platform text normalization such as keyboard auto-capitalization, treat duplicate exact text matches as successful presence assertions, narrow broad selectors by text before reporting ambiguity, reject `text` on non-`textContains` predicates instead of silently ignoring it, and keep recoverable-error candidates compact in structured output.
+- Fixed `tap` on SwiftUI switch element refs by using a touch down/up activation instead of AXe's coordinate tap path.
+- Fixed selector fallback for AXe duplicate-match diagnostics that include parenthesized match counts.
+- Fixed semantic taps and text-field focusing so element refs with duplicate AXe selectors use their resolved snapshot coordinates immediately.
+- Fixed bottom-clipped UI automation targets so taps, touches, and long presses use a visible activation point instead of the hidden center of the accessibility frame.
+- Fixed app-level horizontal swipes so full-screen refs use a content-area y-coordinate instead of missing horizontal carousels by swiping near the hero area.
+- Fixed CLI commands with `simulatorId`-only contracts so `simulatorName` session defaults are resolved to a simulator ID without adding conflicting simulator arguments to tools that already accept `simulatorName`, and fixed simulator lifecycle tools so name-only defaults resolve before simctl operations.
+- Fixed `snapshot_ui` and `wait_for_ui` next steps so they use the resolved simulator ID instead of leaking `SIMULATOR_UUID` placeholders.
+- Fixed the Weather example app so saved-location rows are not reused as search-result rows after editing locations.
+- Fixed the Weather example app's current-location button so it selects the current saved location instead of appearing as a no-op UI automation target.
+- Fixed `type_text` so AXe-unsupported international/accented characters fail before focusing the field, with a clear recoverable error instead of a generic typing failure.
+- Fixed `snapshot_ui` next-step guidance so the suggested tap ref prefers useful tappable controls over text fields, sheet grabbers, close buttons, and clear-search buttons.
+- Fixed compact runtime snapshot JSON so target ordering matches compact text output and prioritizes useful content targets before low-value sheet chrome.
+- Fixed `wait_for_ui` success output so compact text and JSON include the matched elements that satisfied the wait predicate.
+- Fixed `wait_for_ui textContains` so duplicate elements with the same matching visible text satisfy presence-style assertions instead of reporting ambiguity.
+- Fixed CLI `--style minimal` so final text output suppresses generated next steps for daemon-routed tools as intended.
+- Fixed `snapshot_ui` next-step guidance so snapshots with no tappable targets no longer suggest tapping the first non-actionable element.
+- Fixed next-step rendering for tools shared across workflows so follow-up commands prefer the workflow that produced the result instead of drifting to another workflow alias.
+- Fixed `snapshot_ui` next-step guidance so calculator-style utility and operator buttons no longer outrank more useful digit/content controls.
+- Fixed `snapshot_ui` compact text, JSON, and next-step guidance so already-selected segmented controls no longer outrank unselected choices.
+- Fixed compact runtime snapshots and next-step guidance so sheet grabbers remain visible as low-priority targets, allowing agents to expand or dismiss sheets without outranking useful content controls.
+- Fixed compact wait-match rows so static assertion matches render with `none` instead of exposing low-level long-press/touch actions as if they were primary agent actions.
+- Fixed compact runtime snapshot ordering and next-step guidance so destructive controls such as Remove/Delete are demoted behind safer content and navigation targets.
+- Clarified simulator keyboard shortcut failures when Simulator.app is running without a visible device window.
+- Fixed hardware button automation so successful button presses wait briefly for system UI transitions before returning, reducing stale immediate follow-up snapshots.
+- Fixed runtime snapshots so modal sheet hosts remain swipeable after the currently visible sheet content fits inside the viewport.
+- Fixed `wait_for_ui` validation so unknown JSON fields are rejected instead of silently broadening waits.
+- Fixed CLI numeric array flags so comma-separated values such as `--key-codes 23,18,14` are parsed as numbers instead of failing validation.
+- Fixed runtime snapshots so unlabeled internal custom-action nodes, such as SpringBoard icon subviews, are no longer advertised as likely tap targets.
+- Fixed AXe bundling so downloaded artifacts must report the pinned AXe version, and dirty local AXe builds require an explicit opt-in.
+- Fixed runtime snapshot tips so compact output names all target-ref action tools, including `long_press` and `touch`.
+- Clarified key press and key sequence tool descriptions so agents know key codes are AXe/macOS virtual key codes and should prefer `type_text` for text entry.
+- Clarified `wait_for_ui` timeout recovery hints so agents know selector fields match exact values and should use `textContains` for partial visible text.
+- Fixed UI action success next steps so agents are prompted to refresh runtime snapshots before reusing element refs after actions such as swipes.
+- Fixed `snapshot_ui` next-step guidance so state-changing controls such as segmented units and switches remain available in targets without being promoted as generic tap or batch suggestions.
+- Fixed `snapshot_ui` tap next-step priority so content-rich cards are suggested before navigation controls like Settings.
+- Fixed successful UI action results so they include a fresh runtime snapshot and actionable next steps, reducing follow-up refresh calls after taps, typing, swipes, and batches.
+- Fixed same-simulator UI automation transactions so runtime snapshot resolution, actions, invalidation, and refreshes cannot interleave within one MCP or daemon process.
 
 ## [2.5.2]
 
@@ -641,5 +704,3 @@ Please note that the UI automation features are an early preview and currently i
 ## [v1.0.1] - 2025-04-02
 - Initial release of XcodeBuildMCP
 - Basic support for building iOS and macOS applications
-
-

CLAUDE.md

@@ -18,6 +18,7 @@ When reading issues:
 -
 ## Tools
 - GitHub CLI for issues/PRs
+- MCP `readOnlyHint` describes whether a tool mutates host/project state such as files, build artifacts, configuration, or external services. Simulator HID/UI actions that only tap, type, press, or gesture inside the simulator may remain `readOnlyHint: true`; do not flip them to `false` merely because app UI state changes.
 - CLI design note: do not rely on CLI session-default writes. CLI is intentionally deterministic for CI/scripting and should use explicit command arguments as the primary input surface.
 - When working on skill sources in `skills/`, use the `skill-creator` skill workflow.
 - After modifying any skill source, run `npx skill-check <skill-directory>` and address all errors/warnings before handoff.

example_projects/.xcodebuildmcp/config.yaml

@@ -4,13 +4,11 @@ sessionDefaultsProfiles:
     workspacePath: ./iOS_Calculator/CalculatorApp.xcworkspace
     scheme: CalculatorApp
     simulatorName: iPhone 17 Pro
-    simulatorId: B38FE93D-578B-454B-BE9A-C6FA0CE5F096
     simulatorPlatform: iOS Simulator
   ios-test:
     projectPath: ./iOS/MCPTest.xcodeproj
     scheme: MCPTest
     simulatorName: iPhone 17 Pro
-    simulatorId: B38FE93D-578B-454B-BE9A-C6FA0CE5F096
     simulatorPlatform: iOS Simulator
   macos-test:
     projectPath: ./macOS/MCPTest.xcodeproj

example_projects/Weather/.xcodebuildmcp/config.yaml

@@ -7,7 +7,7 @@ sentryDisabled: false
 sessionDefaults:
   projectPath: Weather.xcodeproj
   scheme: Weather
-  simulatorName: iPhone 17 Pro
+  simulatorName: iPhone 17 Pro Max
 setupPreferences:
   platforms:
     - iOS

example_projects/Weather/README.md

@@ -2,22 +2,14 @@
 
 Atmos Weather is a native SwiftUI weather app prototype for iOS.
 
-## Launch with mock weather data
+## Launch
 
-Build and run the app with XcodeBuildMCP first:
+Build and run the app with XcodeBuildMCP:
 
 ```bash
 ../../build/cli.js simulator build-and-run
 ```
 
-Then relaunch the installed app with the mock API argument:
-
-```bash
-../../build/cli.js simulator launch-app \
-  --bundle-id com.sentry.weather.Weather \
-  --args=--mock-weather-api
-```
-
 ## JSON fixtures
 
 Fixture JSON files live in:
@@ -98,4 +90,4 @@ Run the app test suite through XcodeBuildMCP:
 ../../build/cli.js simulator test
 ```
 
-UI tests inject `--mock-weather-api` themselves so they do not depend on the production API endpoint.
+The app uses bundled deterministic weather data so UI tests do not depend on the production API endpoint.

example_projects/Weather/Weather/Services/MockWeatherAPIClient.swift

@@ -29,8 +29,10 @@ struct MockWeatherAPIClient: WeatherAPIClient, Sendable {
         guard !trimmed.isEmpty else { return [] }
 
         let needle = trimmed.localizedLowercase
-        return fixtures.searchPool.filter { location in
-            location.name.localizedLowercase.contains(needle)
+        var seenLocationIDs = Set<WeatherLocation.ID>()
+        return (fixtures.locations + fixtures.searchPool).filter { location in
+            guard seenLocationIDs.insert(location.id).inserted else { return false }
+            return location.name.localizedLowercase.contains(needle)
                 || location.subtitle.localizedLowercase.contains(needle)
                 || (location.country?.localizedLowercase.contains(needle) ?? false)
         }

example_projects/Weather/Weather/Views/Overlays/LocationPickerView.swift

@@ -103,7 +103,7 @@ struct LocationPickerView: View {
     }
 
     private var currentLocationButton: some View {
-        Button(action: {}) {
+        Button(action: selectCurrentLocation) {
             HStack(spacing: 12) {
                 Image(systemName: "location.fill")
                     .font(.system(size: 14))
@@ -145,6 +145,7 @@ struct LocationPickerView: View {
                             onSelect: { select(location) },
                             onRemove: { remove(location) }
                         )
+                        .id("saved-\(location.id)-\(isEditing)")
                     }
                 } else if isLoading {
                     ForEach(0..<3, id: \.self) { _ in SearchSkeletonRow() }
@@ -160,6 +161,7 @@ struct LocationPickerView: View {
                             onPreview: { preview(location) },
                             onAdd: { add(location) }
                         )
+                        .id("search-\(location.id)-\(isSaved(location))-\(justAddedID == location.id)")
                     }
                 }
             }
@@ -229,6 +231,11 @@ struct LocationPickerView: View {
         justAddedID = location.id
     }
 
+    private func selectCurrentLocation() {
+        guard let currentLocation = savedLocations.first else { return }
+        select(currentLocation)
+    }
+
     private func clearAddedIndicator() async {
         guard let id = justAddedID else { return }
         try? await Task.sleep(for: .milliseconds(1_400))

example_projects/Weather/Weather/Views/Overlays/LocationRows.swift

@@ -96,6 +96,7 @@ struct SearchLocationRow: View {
                 .frame(maxWidth: .infinity, alignment: .leading)
             }
             .buttonStyle(.plain)
+            .accessibilityValue(saved || added ? "saved" : "not saved")
 
             VStack(alignment: .trailing, spacing: 3) {
                 Text(WeatherUnitFormatter.temperatureString(location.temperatureC, units: units))