feat: add maestro replay compatibility (#561)

Author: thymikee

CONTRIBUTING.md

@@ -34,11 +34,49 @@ Run tests:
 pnpm test
 ```
 
+Targeted checks:
+
+```bash
+pnpm check:quick
+pnpm check:unit
+pnpm exec vitest run src/compat/maestro/__tests__/replay-flow.test.ts src/compat/__tests__/replay-input.test.ts
+```
+
 Optional device selectors for tests:
 
 - `ANDROID_DEVICE=Pixel_9_Pro_XL` or `ANDROID_SERIAL=emulator-5554`
 - `IOS_DEVICE="iPhone 17 Pro"` or `IOS_UDID=<udid>`
 
+## Test App and Maestro Compatibility
+
+The Expo test app lives in `examples/test-app`. Install its dependencies once:
+
+```bash
+pnpm test-app:install
+```
+
+For Maestro compatibility, we currently have 15 parser/compat unit tests and one
+top-level test-app Maestro flow, `examples/test-app/maestro/checkout-form.yaml`,
+which includes `examples/test-app/maestro/helpers/open-checkout-form.yaml`.
+
+Run only the parser/compat tests:
+
+```bash
+pnpm exec vitest run src/compat/maestro/__tests__/replay-flow.test.ts src/compat/__tests__/replay-input.test.ts
+```
+
+Run the Expo test-app flow on iOS:
+
+```bash
+pnpm test-app:start
+pnpm ad --session test-app-maestro open "Expo Go" exp://127.0.0.1:8081 --platform ios --device "iPhone 17 Pro"
+pnpm ad --session test-app-maestro wait "Agent Device Tester" 30000 --platform ios --device "iPhone 17 Pro"
+pnpm test-app:maestro:ios -- --session test-app-maestro -- --device "iPhone 17 Pro"
+```
+
+Use `pnpm test-app:maestro:android` for Android, passing the same extra
+`agent-device` flags after `--`.
+
 ## Guidelines
 
 - Keep dependencies minimal.

examples/test-app/README.md

@@ -76,3 +76,48 @@ pnpm android
 ```
 
 Once the app is running, use `agent-device` against `Agent Device Tester` like any other target app.
+
+## Local Agent Device suites
+
+The repo includes two local suites for iterating on the fixture app:
+
+```bash
+pnpm test-app:replay:ios
+pnpm test-app:replay:android
+```
+
+These run the `.ad` replay suite in `examples/test-app/replays`.
+
+To target a specific iOS simulator or an installed Expo development build, run the
+underlying command directly so global flags stay before replay inputs:
+
+```bash
+node bin/agent-device.mjs test examples/test-app/replays \
+  --platform ios \
+  --device "iPhone 17 Pro" \
+  --env APP_TARGET=dev.expo.easagentdevice \
+  --env APP_URL=<project-url> \
+  --artifacts-dir .tmp/test-app-replay/ios
+```
+
+Use `APP_TARGET=com.callstack.agentdevicelab` when the standalone fixture app is
+installed instead of an Expo development shell.
+
+The Maestro prototype suite lives in `examples/test-app/maestro` and runs through
+`agent-device replay --maestro`:
+
+```bash
+pnpm test-app:maestro:ios -- --open "Agent Device Tester"
+pnpm test-app:maestro:android -- --open "Agent Device Tester"
+```
+
+When running through Expo Go, start the project first and pass the shell that is already showing
+the app, for example:
+
+```bash
+pnpm test-app:maestro:ios -- --open "Expo Go"
+```
+
+The suite intentionally covers the compat layer syntax used by public Maestro suites:
+`runFlow` file/inline blocks, `when.platform`, config hooks, deterministic `repeat.times`,
+flow `env`, selectors, input, assertions, and swipe.

examples/test-app/maestro/checkout-form.yaml

@@ -0,0 +1,44 @@
+appId: host.exp.Exponent
+env:
+  CHECKOUT_NAME: Ada Lovelace
+  CHECKOUT_EMAIL: ada@example.com
+  PICKUP_TAPS: "2"
+onFlowStart:
+  - assertVisible: Agent Device Tester
+onFlowComplete:
+  - assertVisible: Delivery choices
+---
+- runFlow:
+    file: helpers/open-checkout-form.yaml
+    env:
+      CHECKOUT_NAME: Ada Lovelace
+      CHECKOUT_EMAIL: ada@example.com
+- runFlow:
+    when:
+      platform: iOS
+    commands:
+      - assertVisible: Checkout form
+- runFlow:
+    when:
+      platform: Android
+    commands:
+      - assertVisible: Checkout form
+- assertVisible:
+    text: Checkout form
+- swipe:
+    start: 50%, 75%
+    end: 50%, 35%
+    duration: 300
+- repeat:
+    times: ${PICKUP_TAPS}
+    commands:
+      - tapOn:
+          id: shipping-pickup
+      - assertVisible:
+          id: shipping-pickup
+          selected: true
+- tapOn:
+    id: payment-cash
+- assertVisible:
+    id: payment-cash
+    selected: true

examples/test-app/maestro/helpers/open-checkout-form.yaml

@@ -0,0 +1,14 @@
+---
+- tapOn:
+    id: home-open-form
+- assertVisible: Checkout form
+- tapOn:
+    id: field-name
+- inputText:
+    text: ${CHECKOUT_NAME}
+    label: Full name
+- tapOn:
+    id: field-email
+- inputText: ${CHECKOUT_EMAIL}
+- tapOn:
+    text: Checkout form

examples/test-app/replays/checkout-form-android.ad

@@ -0,0 +1,21 @@
+context platform=android timeout=60000
+env APP_TARGET="Agent Device Tester"
+env APP_URL=""
+open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}"
+wait "label=\"Form\"" 30000
+click "label=\"Form\""
+wait "Checkout form" 5000
+fill id="field-name" "Ada Lovelace"
+fill id="field-email" "ada@example.com"
+keyboard dismiss
+wait "Checkout form" 5000
+scroll down 0.6
+click id="shipping-pickup"
+click id="payment-cash"
+wait "Delivery choices" 5000
+scroll down 0.7
+click id="checkbox-agree"
+click id="submit-order"
+wait "Order summary" 5000
+wait "Ada Lovelace chose pickup with cash payment." 5000
+close

examples/test-app/replays/checkout-form.ad

@@ -0,0 +1,21 @@
+context platform=ios timeout=60000
+env APP_TARGET="Agent Device Tester"
+env APP_URL=""
+open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}"
+wait "label=\"Form\"" 30000
+click "label=\"Form\""
+wait "Checkout form" 5000
+fill id="field-name" "Ada Lovelace"
+fill id="field-email" "ada@example.com"
+keyboard dismiss
+wait "Checkout form" 5000
+scroll down 0.6
+click id="shipping-pickup"
+click id="payment-cash"
+wait "Delivery choices" 5000
+scroll down 0.7
+click id="checkbox-agree"
+click id="submit-order"
+wait "Order summary" 5000
+wait "Ada Lovelace chose pickup with cash payment." 5000
+close

package.json

@@ -111,6 +111,11 @@
     "test-app:ios": "pnpm --dir examples/test-app ios",
     "test-app:android": "pnpm --dir examples/test-app android",
     "test-app:typecheck": "pnpm --dir examples/test-app typecheck",
+    "test-app:replay:ios": "pnpm ad test examples/test-app/replays --platform ios --artifacts-dir .tmp/test-app-replay/ios",
+    "test-app:replay:android": "pnpm ad test examples/test-app/replays --platform android --artifacts-dir .tmp/test-app-replay/android",
+    "test-app:maestro": "node scripts/run-test-app-maestro-suite.mjs",
+    "test-app:maestro:ios": "node scripts/run-test-app-maestro-suite.mjs --platform ios",
+    "test-app:maestro:android": "node scripts/run-test-app-maestro-suite.mjs --platform android",
     "test": "vitest run --project unit",
     "test:unit": "vitest run --project unit",
     "test:coverage": "vitest run --coverage",

scripts/run-test-app-maestro-suite.mjs

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const binPath = path.join(repoRoot, 'bin', 'agent-device.mjs');
+
+const options = {
+  platform: 'ios',
+  session: 'test-app-maestro',
+  flowDir: path.join(repoRoot, 'examples', 'test-app', 'maestro'),
+  openTarget: '',
+  close: false,
+  passthrough: [],
+};
+
+for (let index = 2; index < process.argv.length; index += 1) {
+  const arg = process.argv[index];
+  if (arg === '--') {
+    options.passthrough.push(...process.argv.slice(index + 1));
+    break;
+  }
+  if (arg === '--platform' && process.argv[index + 1]) {
+    options.platform = process.argv[index + 1];
+    index += 1;
+    continue;
+  }
+  if (arg === '--session' && process.argv[index + 1]) {
+    options.session = process.argv[index + 1];
+    index += 1;
+    continue;
+  }
+  if (arg === '--flow-dir' && process.argv[index + 1]) {
+    options.flowDir = path.resolve(process.argv[index + 1]);
+    index += 1;
+    continue;
+  }
+  if (arg === '--open' && process.argv[index + 1]) {
+    options.openTarget = process.argv[index + 1];
+    index += 1;
+    continue;
+  }
+  if (arg === '--close') {
+    options.close = true;
+    continue;
+  }
+  options.passthrough.push(arg);
+}
+
+const flows = fs
+  .readdirSync(options.flowDir)
+  .filter((entry) => entry.endsWith('.yaml') || entry.endsWith('.yml'))
+  .sort()
+  .map((entry) => path.join(options.flowDir, entry));
+
+if (flows.length === 0) {
+  console.error(`No Maestro flows found in ${options.flowDir}`);
+  process.exit(1);
+}
+
+function runAgentDevice(args) {
+  execFileSync(process.execPath, [binPath, '--session', options.session, ...args], {
+    cwd: repoRoot,
+    stdio: 'inherit',
+  });
+}
+
+if (options.openTarget) {
+  runAgentDevice(['open', options.openTarget, '--platform', options.platform, ...options.passthrough]);
+  runAgentDevice(['wait', 'Agent Device Tester', '30000', '--platform', options.platform, ...options.passthrough]);
+}
+
+for (const flow of flows) {
+  runAgentDevice(['replay', flow, '--maestro', '--platform', options.platform, ...options.passthrough]);
+}
+
+if (options.close) {
+  runAgentDevice(['close']);
+}

src/__tests__/client.test.ts

@@ -399,6 +399,7 @@ test('client throws AppError for daemon failures', async () => {
   );
 });
 
+// fallow-ignore-next-line complexity
 test('replay.run serializes client-collected AD_VAR shell env into daemon request', async () => {
   const previousAppId = process.env.AD_VAR_APP_ID;
   const previousWaitMs = process.env.AD_VAR_WAIT_MS;
@@ -419,6 +420,7 @@ test('replay.run serializes client-collected AD_VAR shell env into daemon reques
     assert.equal(setup.calls[0]?.command, 'replay');
     assert.deepEqual(setup.calls[0]?.positionals, ['./flows/login.ad']);
     assert.deepEqual(setup.calls[0]?.flags?.replayEnv, ['APP_ID=cli-override']);
+    assert.equal(setup.calls[0]?.flags?.replayBackend, undefined);
     const replayShellEnv = setup.calls[0]?.flags?.replayShellEnv as
       | Record<string, string>
       | undefined;
@@ -435,6 +437,36 @@ test('replay.run serializes client-collected AD_VAR shell env into daemon reques
   }
 });
 
+test('replay.run forwards backend without knowing the concrete syntax', async () => {
+  const setup = createTransport(async () => ({ ok: true, data: {} }));
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  await client.replay.run({
+    path: './flows/login.yaml',
+    backend: 'external-flow',
+  });
+
+  assert.equal(setup.calls.length, 1);
+  assert.equal(setup.calls[0]?.command, 'replay');
+  assert.deepEqual(setup.calls[0]?.positionals, ['./flows/login.yaml']);
+  assert.equal(setup.calls[0]?.flags?.replayBackend, 'external-flow');
+});
+
+test('replay.run keeps deprecated maestro option as backend alias', async () => {
+  const setup = createTransport(async () => ({ ok: true, data: {} }));
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  await client.replay.run({
+    path: './flows/login.yaml',
+    maestro: true,
+  });
+
+  assert.equal(setup.calls.length, 1);
+  assert.equal(setup.calls[0]?.command, 'replay');
+  assert.deepEqual(setup.calls[0]?.positionals, ['./flows/login.yaml']);
+  assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro');
+});
+
 test('client.command.wait prepares selector options and rejects invalid selectors', async () => {
   const setup = createTransport(async () => ({
     ok: true,

src/cli/commands/generic.ts

@@ -58,7 +58,7 @@ const genericClientCommandRunners = {
       ...buildSelectionOptions(flags),
       path: required(positionals[0], 'replay requires path'),
       update: flags.replayUpdate,
-      maestro: flags.replayMaestro,
+      backend: flags.replayMaestro ? 'maestro' : undefined,
       env: flags.replayEnv,
     }),
   test: ({ client, positionals, flags }) => {