Skip to content

Commit f71371e

Browse files
authored
fix: handle platform alerts (#562)
* fix: handle Android platform alerts * fix: handle iOS alerts in runner * chore: type alert metadata * refactor: colocate Android alert handling
1 parent 896adcc commit f71371e

24 files changed

Lines changed: 1328 additions & 185 deletions
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import XCTest
2+
3+
extension RunnerTests {
4+
struct RunnerAlert {
5+
let root: XCUIElement
6+
let ownerApp: XCUIApplication
7+
let buttons: [XCUIElement]
8+
}
9+
10+
func resolveAlert(app activeApp: XCUIApplication) -> RunnerAlert? {
11+
if let alert = firstExistingElement(in: activeApp.alerts.allElementsBoundByIndex) {
12+
return runnerAlert(root: alert, ownerApp: activeApp)
13+
}
14+
if let popup = firstDismissPopupWindow(in: activeApp) {
15+
return runnerAlert(root: popup, ownerApp: activeApp)
16+
}
17+
#if os(macOS)
18+
return nil
19+
#else
20+
if let systemModal = firstBlockingSystemModal(in: springboard) {
21+
return runnerAlert(root: systemModal, ownerApp: springboard)
22+
}
23+
return nil
24+
#endif
25+
}
26+
27+
func handleAlert(_ alert: RunnerAlert, action: String) -> Response {
28+
if action == "accept" || action == "dismiss" {
29+
guard let button = chooseAlertButton(alert.buttons, action: action) else {
30+
return Response(ok: false, error: ErrorPayload(message: "alert \(action) button not found"))
31+
}
32+
let outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)")
33+
if let response = unsupportedResponse(for: outcome) {
34+
return response
35+
}
36+
return Response(ok: true, data: DataPayload(message: action == "accept" ? "accepted" : "dismissed"))
37+
}
38+
39+
return Response(
40+
ok: true,
41+
data: DataPayload(
42+
message: preferredAlertTitle(alert.root, buttons: alert.buttons),
43+
items: alert.buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) }
44+
)
45+
)
46+
}
47+
48+
private func runnerAlert(root: XCUIElement, ownerApp: XCUIApplication) -> RunnerAlert? {
49+
let buttons = actionableElements(in: root).filter { isEnabledElement($0) }
50+
guard !buttons.isEmpty else {
51+
return nil
52+
}
53+
return RunnerAlert(root: root, ownerApp: ownerApp, buttons: buttons)
54+
}
55+
56+
private func firstExistingElement(in elements: [XCUIElement]) -> XCUIElement? {
57+
elements.first { isVisibleElement($0) }
58+
}
59+
60+
private func firstDismissPopupWindow(in app: XCUIApplication) -> XCUIElement? {
61+
safeElementsQuery {
62+
app.windows.allElementsBoundByIndex
63+
}.first { window in
64+
if !isVisibleElement(window) { return false }
65+
if isDismissPopupMarker(window.label) || isDismissPopupMarker(window.identifier) {
66+
return true
67+
}
68+
return safeElementsQuery {
69+
window.descendants(matching: .any).allElementsBoundByIndex
70+
}.contains { descendant in
71+
isDismissPopupMarker(descendant.label) || isDismissPopupMarker(descendant.identifier)
72+
}
73+
}
74+
}
75+
76+
private func chooseAlertButton(_ buttons: [XCUIElement], action: String) -> XCUIElement? {
77+
if action == "accept" {
78+
if let accept = buttons.first(where: { isAcceptButton($0.label) }) {
79+
return accept
80+
}
81+
return buttons.count == 1 && !isDismissButton(buttons[0].label) ? buttons[0] : nil
82+
}
83+
84+
return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last
85+
}
86+
87+
private func isAcceptButton(_ label: String) -> Bool {
88+
let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
89+
return [
90+
"ok",
91+
"allow",
92+
"yes",
93+
"continue",
94+
"done",
95+
"open settings"
96+
].contains(normalized) || normalized.hasPrefix("confirm")
97+
}
98+
99+
private func isDismissButton(_ label: String) -> Bool {
100+
[
101+
"cancel",
102+
"close",
103+
"dismiss",
104+
"don't allow",
105+
"don’t allow",
106+
"not now",
107+
"no",
108+
"keep browsing",
109+
"later"
110+
].contains(label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased())
111+
}
112+
113+
private func preferredAlertTitle(_ element: XCUIElement, buttons: [XCUIElement]) -> String {
114+
let buttonLabels = Set(buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() })
115+
let descendants = element.descendants(matching: .any).allElementsBoundByIndex
116+
for descendant in descendants {
117+
let text = descendant.label.trimmingCharacters(in: .whitespacesAndNewlines)
118+
if text.isEmpty ||
119+
isGenericAlertLabel(text) ||
120+
buttonLabels.contains(text.lowercased()) ||
121+
descendant.elementType == .navigationBar ||
122+
actionableTypes.contains(descendant.elementType)
123+
{
124+
continue
125+
}
126+
return text
127+
}
128+
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
129+
return label.isEmpty || isGenericAlertLabel(label) ? "Alert" : label
130+
}
131+
132+
private func isGenericAlertLabel(_ label: String) -> Bool {
133+
let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
134+
return isDismissPopupMarker(normalized) ||
135+
normalized.hasPrefix("vertical scroll bar") ||
136+
normalized.hasPrefix("horizontal scroll bar") ||
137+
normalized == "tab bar"
138+
}
139+
140+
private func isVisibleElement(_ element: XCUIElement) -> Bool {
141+
element.exists && !element.frame.isNull && !element.frame.isEmpty
142+
}
143+
144+
private func isEnabledElement(_ element: XCUIElement) -> Bool {
145+
var enabled = false
146+
_ = RunnerObjCExceptionCatcher.catchException({
147+
enabled = element.exists && element.isEnabled
148+
})
149+
return enabled
150+
}
151+
152+
private func isDismissPopupMarker(_ label: String) -> Bool {
153+
label.trimmingCharacters(in: .whitespacesAndNewlines).caseInsensitiveCompare("dismiss popup") == .orderedSame
154+
}
155+
}

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ extension RunnerTests {
1313
return (gestureStartUptimeMs, currentUptimeMs())
1414
}
1515

16-
private func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
16+
func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? {
1717
switch outcome {
1818
case .performed:
1919
return nil
@@ -731,32 +731,10 @@ extension RunnerTests {
731731
)
732732
case .alert:
733733
let action = (command.action ?? "get").lowercased()
734-
let alert = activeApp.alerts.firstMatch
735-
if !alert.exists {
734+
guard let alert = resolveAlert(app: activeApp) else {
736735
return Response(ok: false, error: ErrorPayload(message: "alert not found"))
737736
}
738-
if action == "accept" {
739-
guard let button = alert.buttons.allElementsBoundByIndex.first else {
740-
return Response(ok: false, error: ErrorPayload(message: "alert accept button not found"))
741-
}
742-
let outcome = activateElement(app: activeApp, element: button, action: "alert accept")
743-
if let response = unsupportedResponse(for: outcome) {
744-
return response
745-
}
746-
return Response(ok: true, data: DataPayload(message: "accepted"))
747-
}
748-
if action == "dismiss" {
749-
guard let button = alert.buttons.allElementsBoundByIndex.last else {
750-
return Response(ok: false, error: ErrorPayload(message: "alert dismiss button not found"))
751-
}
752-
let outcome = activateElement(app: activeApp, element: button, action: "alert dismiss")
753-
if let response = unsupportedResponse(for: outcome) {
754-
return response
755-
}
756-
return Response(ok: true, data: DataPayload(message: "dismissed"))
757-
}
758-
let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label }
759-
return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels))
737+
return handleAlert(alert, action: action)
760738
case .pinch:
761739
guard let scale = command.scale, scale > 0 else {
762740
return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0"))

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ extension RunnerTests {
4646
#endif
4747
}
4848

49-
private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
49+
func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? {
5050
let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE")
5151
let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in
5252
if disableSafeProbe {
@@ -76,7 +76,7 @@ extension RunnerTests {
7676
return nil
7777
}
7878

79-
private func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] {
79+
func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] {
8080
var elements: [XCUIElement] = []
8181
let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
8282
elements = fetch()
@@ -120,7 +120,7 @@ extension RunnerTests {
120120
return true
121121
}
122122

123-
private func actionableElements(in element: XCUIElement) -> [XCUIElement] {
123+
func actionableElements(in element: XCUIElement) -> [XCUIElement] {
124124
var seen = Set<String>()
125125
var actions: [XCUIElement] = []
126126
let descendants = safeElementsQuery {

src/alert-contract.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
export const ALERT_POLL_INTERVAL_MS = 300;
2+
export const DEFAULT_ALERT_TIMEOUT_MS = 10_000;
3+
export const ALERT_ACTION_RETRY_MS = 2_000;
4+
5+
export type AlertAction = 'get' | 'accept' | 'dismiss' | 'wait';
6+
7+
export type AlertPlatform = 'android' | 'ios' | 'macos';
8+
9+
export type AlertSource = 'permission' | 'native-dialog' | 'system-dialog';
10+
11+
export type AlertInfo = {
12+
title?: string;
13+
message?: string;
14+
buttons?: string[];
15+
platform?: AlertPlatform;
16+
source?: AlertSource;
17+
packageName?: string;
18+
};

src/backend.ts

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { AndroidSnapshotBackendMetadata } from './platforms/android/snapshot-types.ts';
2+
import type { AlertAction, AlertInfo } from './alert-contract.ts';
23
import type { AppsFilter } from './commands/app-inventory-contract.ts';
34
import type {
45
Point,
@@ -110,13 +111,9 @@ export type BackendClipboardTextResult = {
110111
text: string;
111112
};
112113

113-
export type BackendAlertAction = 'get' | 'accept' | 'dismiss' | 'wait';
114+
export type BackendAlertAction = AlertAction;
114115

115-
export type BackendAlertInfo = {
116-
title?: string;
117-
message?: string;
118-
buttons?: string[];
119-
};
116+
export type BackendAlertInfo = AlertInfo;
120117

121118
export type BackendAlertResult =
122119
| {

src/client-types.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ import type {
2424
import type { MetroBridgeScope } from './client-companion-tunnel-contract.ts';
2525
import type { AppsFilter } from './commands/app-inventory-contract.ts';
2626
import type { ScreenshotRequestFlags } from './commands/capture-screenshot-options.ts';
27+
import type { AlertInfo } from './alert-contract.ts';
2728

2829
export type { FindLocator } from './utils/finders.ts';
2930
export type { CompanionTunnelScope, MetroBridgeScope } from './client-companion-tunnel-contract.ts';
3031
export type { AppsFilter } from './commands/app-inventory-contract.ts';
32+
export type { AlertAction, AlertInfo, AlertPlatform, AlertSource } from './alert-contract.ts';
3133

3234
type DaemonTransportMode = 'auto' | 'socket' | 'http';
3335
type DaemonServerMode = 'socket' | 'http' | 'dual';
@@ -404,7 +406,19 @@ export type WaitCommandResult = DaemonResponseData & {
404406
selector?: string;
405407
};
406408

407-
export type AlertCommandResult = DaemonResponseData;
409+
export type AlertCommandResult = DaemonResponseData & {
410+
kind?: 'alertStatus' | 'alertHandled' | 'alertWait';
411+
action?: AlertCommandOptions['action'];
412+
alert?: AlertInfo | null;
413+
handled?: boolean;
414+
button?: string;
415+
waitedMs?: number;
416+
timedOut?: boolean;
417+
platform?: AlertInfo['platform'];
418+
accepted?: boolean;
419+
dismissed?: boolean;
420+
items?: string[];
421+
};
408422

409423
type CommandActionResult<T extends string> = DaemonResponseData & {
410424
action?: T;

src/core/__tests__/capabilities.test.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,23 @@ function assertCommandSupport(commands: string[], checks: SupportCheck[]): void
7777
test('device capability matrix stays consistent across shared command groups', () => {
7878
const scenarios: Array<{ commands: string[]; checks: SupportCheck[] }> = [
7979
{
80-
commands: ['alert', 'pinch'],
80+
commands: ['pinch'],
8181
checks: [
8282
{ device: iosSimulator, expected: true, label: 'on iOS sim' },
8383
{ device: iosDevice, expected: false, label: 'on iOS device' },
8484
{ device: androidDevice, expected: false, label: 'on Android' },
8585
{ device: macOsDevice, expected: true, label: 'on macOS' },
8686
],
8787
},
88+
{
89+
commands: ['alert'],
90+
checks: [
91+
{ device: iosSimulator, expected: true, label: 'on iOS sim' },
92+
{ device: iosDevice, expected: false, label: 'on iOS device' },
93+
{ device: androidDevice, expected: true, label: 'on Android' },
94+
{ device: macOsDevice, expected: true, label: 'on macOS' },
95+
],
96+
},
8897
{
8998
commands: ['settings', 'clipboard'],
9099
checks: [

src/core/capabilities.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ const COMMAND_CAPABILITY_MATRIX: Record<string, CommandCapability> = {
3535
// macOS desktop targets report kind=device, so this stays enabled here and the
3636
// supports() guard excludes iOS physical devices.
3737
apple: { simulator: true, device: true },
38-
android: {},
38+
android: { emulator: true, device: true, unknown: true },
3939
linux: LINUX_NONE,
40-
supports: isMacOsOrAppleSimulator,
40+
supports: (device) => device.platform === 'android' || isMacOsOrAppleSimulator(device),
4141
},
4242
pinch: {
4343
// macOS desktop targets report kind=device, so this stays enabled here and the

src/daemon/handlers/parse-utils.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
export const POLL_INTERVAL_MS = 300;
2-
export const DEFAULT_TIMEOUT_MS = 10_000;
1+
export {
2+
ALERT_ACTION_RETRY_MS,
3+
ALERT_POLL_INTERVAL_MS as POLL_INTERVAL_MS,
4+
DEFAULT_ALERT_TIMEOUT_MS as DEFAULT_TIMEOUT_MS,
5+
} from '../../alert-contract.ts';
36

47
export function parseTimeout(value: string | undefined): number | null {
58
if (!value) return null;

0 commit comments

Comments
 (0)