fix: make layer-3 scenarios prove what they claim, and parse the Maestro version

Run 3 (29497919702) got the whole device path working: Expo build (30m), app
installed, simctl check, pinned Maestro CLI install. Only the version ASSERTION
failed — `maestro --version` prints an analytics banner before the version, and
`tr -d '[:space:]'` mashed banner+version into one string. The CLI was correctly
2.5.1. Match the semver line instead, and set MAESTRO_CLI_NO_ANALYTICS (CI should
not phone home). Verified the parse against the exact CI output: banner and clean
forms both yield 2.5.1, wrong/empty still fail.

tap-retry-if-no-change was vacuous: it tapped a navigating control, so the first
tap always succeeded and retryIfNoChange never ran — it passed while proving
nothing. It now taps the app's non-interactive title so the screen cannot change
and the retry path is forced, and asserts tapRetries >= 1 from the trace
(MaestroRuntimeMetrics already records it per step). A new metricAtLeast invariant
kind carries the assertion; a test reproduces the old vacuity.

percent-swipe no longer claims bug class 1. Truncation vs rounding is a <=1px
delta that no app-observable device outcome can distinguish, so pass/pass could
never back that claim up. The runtime half is instead pinned exactly by a pure
unit test of resolveMaestroCoordinate (it short-circuits on a known viewport, so
no device is needed) — verified to catch the regression by flipping trunc->round,
which turns 3 of 6 tests red. Truncation had no test coverage at all before this.
A test now forbids any device scenario from re-claiming bug class 1.

Author: thymikee

.github/workflows/conformance-differential.yml

@@ -43,6 +43,9 @@ concurrency:
 env:
   AGENT_DEVICE_CLI: "--experimental-strip-types src/bin.ts"
   DIFFERENTIAL_ONLY: ${{ github.event.inputs.only || '' }}
+  # CI should not phone home, and it keeps `maestro --version` to just the
+  # version instead of prefixing an analytics notice.
+  MAESTRO_CLI_NO_ANALYTICS: "1"
 
 jobs:
   differential-ios:
@@ -115,7 +118,10 @@ jobs:
         run: |
           set -euo pipefail
           EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
-          ACTUAL=$(maestro --version 2>/dev/null | tr -d '[:space:]')
+          # `maestro --version` can print notices (e.g. the analytics banner)
+          # around the version, so match the semver line rather than the whole
+          # output. Empty ACTUAL fails the comparison below, as it should.
+          ACTUAL=$(maestro --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | tail -1 || true)
           echo "expected=$EXPECTED actual=$ACTUAL"
           if [ "$ACTUAL" != "$EXPECTED" ]; then
             echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior."

scripts/maestro-conformance/differential/flows/tap-retry-if-no-change.yaml

@@ -1,11 +1,15 @@
-# Layer-3 device flow. A tap whose screen may not change exercises the
-# retryIfNoChange path on both engines.
+# Layer-3 device flow. Taps the app's static title, which is NOT interactive, so
+# the screen cannot change and the retryIfNoChange path is forced to run. A tap on
+# a navigating control (e.g. home-open-form) succeeds on the first attempt and
+# never exercises retry at all — the scenario would pass while proving nothing.
+# The scenario asserts tapRetries >= 1 from the trace, so a retry regression is
+# caught rather than assumed.
 appId: com.callstack.agentdevicelab
 ---
 - launchApp:
     clearState: true
 - assertVisible: Agent Device Tester
 - tapOn:
-    id: home-open-form
+    text: Agent Device Tester
     retryTapIfNoChange: true
-- assertVisible: Checkout form
+- assertVisible: Agent Device Tester

scripts/maestro-conformance/differential/invariants.test.ts

@@ -78,6 +78,66 @@ test('readTrace on a missing file returns no events', () => {
 
 test('bug class 4 has a machine-checkable invariant, not just outcome parity', () => {
   const settle = DIFFERENTIAL_SCENARIOS.find((scenario) => scenario.bugClass === 4);
-  assert.ok(settle?.engineInvariants?.length, 'settle scenario must carry an engine-side invariant');
-  assert.equal(settle?.engineInvariants?.[0]?.maxMs, MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS);
+  const invariant = settle?.engineInvariants?.[0];
+  assert.ok(invariant, 'settle scenario must carry an engine-side invariant');
+  assert.equal(invariant?.kind, 'stepDurationBelow');
+  assert.equal(
+    invariant?.kind === 'stepDurationBelow' ? invariant.maxMs : undefined,
+    MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS,
+  );
+});
+
+// --- metricAtLeast: proves a code path actually ran, not just that it passed ---
+
+const RETRY_INVARIANT: Invariant = {
+  kind: 'metricAtLeast',
+  command: 'tapOn',
+  metric: 'tapRetries',
+  min: 1,
+  because: 'test',
+};
+
+const stopWithMetrics = (command: string, metrics: Record<string, number>, step = 1) => ({
+  type: 'replay_action_stop',
+  step,
+  command,
+  ok: true,
+  durationMs: 100,
+  resultTiming: metrics,
+});
+
+test('a tap that actually retried holds the retry invariant', () => {
+  const result = evaluateInvariant([stopWithMetrics('tapOn', { tapRetries: 1 })], RETRY_INVARIANT);
+  assert.equal(result.status, 'held');
+});
+
+// The exact vacuity that made this scenario worthless before: the tap succeeded
+// first try (navigating control), so retry never ran — yet the flow passed.
+test('a tap that never retried violates the invariant (catches a vacuous scenario)', () => {
+  const result = evaluateInvariant([stopWithMetrics('tapOn', { tapRetries: 0 })], RETRY_INVARIANT);
+  assert.equal(result.status, 'violated');
+  assert.match(result.detail, /was 0/);
+});
+
+test('a trace whose taps record no metric reports no-data rather than passing', () => {
+  const result = evaluateInvariant([stopWithMetrics('tapOn', {})], RETRY_INVARIANT);
+  assert.equal(result.status, 'no-data');
+});
+
+test('the retry scenario asserts the retry path ran', () => {
+  const retry = DIFFERENTIAL_SCENARIOS.find((s) => s.id === 'tap-retry-if-no-change');
+  const invariant = retry?.engineInvariants?.[0];
+  assert.ok(invariant, 'retry scenario must prove a retry happened, not assume it');
+  assert.equal(invariant?.kind, 'metricAtLeast');
+});
+
+// Truncation-vs-rounding is at most 1px and cannot be observed on a device, so
+// no scenario may claim to guard bug class 1; a unit test pins it instead.
+test('no device scenario claims to prove percent truncation (bug class 1)', () => {
+  const claiming = DIFFERENTIAL_SCENARIOS.filter((s) => s.bugClass === 1);
+  assert.deepEqual(
+    claiming.map((s) => s.id),
+    [],
+    'a 1px truncation delta is not app-observable; keep bug class 1 in runtime-port-geometry.test.ts',
+  );
 });

scripts/maestro-conformance/differential/invariants.ts

@@ -21,16 +21,27 @@ export type TraceEvent = {
   command?: string;
   ok?: boolean;
   durationMs?: number;
+  /** Per-step MaestroRuntimeMetrics delta (hierarchyCaptures/screenshotCaptures/tapRetries). */
+  resultTiming?: Record<string, unknown>;
 };
 
-export type Invariant = {
-  kind: 'stepDurationBelow';
-  /** Maestro command name the step must match (e.g. "tapOn"). */
-  command: string;
-  maxMs: number;
-  /** Why this bound means the engine behaved correctly. */
-  because: string;
-};
+export type Invariant =
+  | {
+      kind: 'stepDurationBelow';
+      /** Maestro command name the step must match (e.g. "tapOn"). */
+      command: string;
+      maxMs: number;
+      /** Why this bound means the engine behaved correctly. */
+      because: string;
+    }
+  | {
+      kind: 'metricAtLeast';
+      command: string;
+      /** MaestroRuntimeMetrics key, recorded per step as a delta. */
+      metric: 'tapRetries' | 'hierarchyCaptures' | 'screenshotCaptures';
+      min: number;
+      because: string;
+    };
 
 export type InvariantResult = {
   invariant: Invariant;
@@ -56,12 +67,7 @@ export function readTrace(file: string): TraceEvent[] {
 }
 
 function completedSteps(events: TraceEvent[], command: string): TraceEvent[] {
-  return events.filter(
-    (event) =>
-      event.type === 'replay_action_stop' &&
-      event.command === command &&
-      typeof event.durationMs === 'number',
-  );
+  return events.filter((event) => event.type === 'replay_action_stop' && event.command === command);
 }
 
 export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): InvariantResult {
@@ -73,7 +79,43 @@ export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): I
       detail: `no completed ${invariant.command} steps in the trace`,
     };
   }
-  const worst = Math.max(...steps.map((step) => step.durationMs ?? 0));
+
+  if (invariant.kind === 'metricAtLeast') {
+    const values = steps
+      .map((step) => step.resultTiming?.[invariant.metric])
+      .filter((value): value is number => typeof value === 'number');
+    if (values.length === 0) {
+      return {
+        invariant,
+        status: 'no-data',
+        detail: `no ${invariant.command} step recorded a ${invariant.metric} metric`,
+      };
+    }
+    // Per-step deltas: the strongest single step is what proves the path ran.
+    const best = Math.max(...values);
+    if (best < invariant.min) {
+      return {
+        invariant,
+        status: 'violated',
+        detail: `highest ${invariant.command} ${invariant.metric} was ${best} (< ${invariant.min}): ${invariant.because}`,
+      };
+    }
+    return {
+      invariant,
+      status: 'held',
+      detail: `${invariant.command} ${invariant.metric} reached ${best} (>= ${invariant.min})`,
+    };
+  }
+
+  const timed = steps.filter((step) => typeof step.durationMs === 'number');
+  if (timed.length === 0) {
+    return {
+      invariant,
+      status: 'no-data',
+      detail: `no ${invariant.command} step recorded a duration`,
+    };
+  }
+  const worst = Math.max(...timed.map((step) => step.durationMs ?? 0));
   if (worst >= invariant.maxMs) {
     return {
       invariant,

scripts/maestro-conformance/differential/scenarios.ts

@@ -63,20 +63,39 @@ export const DIFFERENTIAL_SCENARIOS: DifferentialScenario[] = [
       'agent-device settled in a different order than upstream (sleep-before vs sleep-after capture) or never latches within the shared budget.',
   },
   {
-    id: 'percent-truncation',
-    bugClass: 1,
+    id: 'percent-swipe',
+    // Deliberately NOT tagged bugClass 1. Truncation-vs-rounding is at most a
+    // one-pixel difference: no app-observable outcome on a real device can
+    // distinguish it, so claiming this scenario guards bug class 1 would be a
+    // lie the pass/pass result cannot back up. That half is pinned exactly by a
+    // pure unit test of the conversion
+    // (src/compat/maestro/__tests__/runtime-port-geometry.test.ts), and the
+    // parse-level half by layer 1 (bug-classes/percent-decimal-swipe). What this
+    // scenario adds is only that percentage swipes work end-to-end on both.
     flow: 'differential/flows/percent-swipe.yaml',
     comparesAcrossEngines:
-      'The percentage swipe completes on both engines. NOTE: this does not compare the resolved pixel coordinates — truncation-vs-rounding is a 1px difference that outcome parity cannot see. Parse-level decimal rejection is covered by layer 1 (bug-classes/percent-decimal-swipe).',
+      'A percentage-endpoint swipe completes on both engines. It does NOT compare resolved pixel endpoints — no metric records them and a 1px delta is not app-observable.',
     expect: 'pass',
     divergenceMeans: 'The swipe behaves differently enough on one engine to fail the flow.',
   },
   {
     id: 'tap-retry-if-no-change',
     flow: 'differential/flows/tap-retry-if-no-change.yaml',
     comparesAcrossEngines:
-      'A retryTapIfNoChange tap succeeds on both engines. NOTE: the retap COUNT is not compared — the trace does not currently expose per-tap attempt counts.',
+      'A retryTapIfNoChange tap succeeds on both engines. The flow taps a non-interactive title so the screen cannot change and the retry path is forced; the invariant below then proves a retry actually happened.',
     expect: 'pass',
+    // Outcome parity would pass even if retry never ran. tapRetries is recorded
+    // per step as a delta (replay-plan-execution.ts), so assert the path executed.
+    engineInvariants: [
+      {
+        kind: 'metricAtLeast',
+        command: 'tapOn',
+        metric: 'tapRetries',
+        min: 1,
+        because:
+          'a tap on an unchanging screen must re-tap; zero retries means retryIfNoChange never ran and the scenario proved nothing',
+      },
+    ],
     divergenceMeans: 'agent-device fails a tap upstream completes (or vice versa) under retry-if-no-change.',
   },
   {

src/compat/maestro/__tests__/runtime-port-geometry.test.ts

@@ -0,0 +1,65 @@
+import assert from 'node:assert/strict';
+import { describe, test } from 'vitest';
+import { resolveMaestroCoordinate } from '../runtime-port-geometry.ts';
+
+// Bug class 1 (#1217), runtime half: upstream Maestro converts percentage
+// coordinates to pixels by integer division (Maestro.kt), so the result is
+// TRUNCATED, never rounded. The parse-level half (rejecting decimal percentages)
+// is covered by the conformance oracle's layer-1 corpus.
+//
+// This is deliberately a unit test rather than a layer-3 device scenario:
+// truncation and rounding differ by at most one pixel, which no app-observable
+// outcome on a real device can distinguish. A pure test of the conversion pins it
+// exactly. `resolveMaestroCoordinate` short-circuits on a known viewport, so no
+// port or device is involved.
+describe('resolveMaestroCoordinate percentage conversion', () => {
+  const viewport = (width: number, height: number, x = 0, y = 0) => ({ x, y, width, height });
+  const resolve = (percent: { x: number; y: number }, vp: ReturnType<typeof viewport>) =>
+    resolveMaestroCoordinate(
+      { space: 'percent', x: percent.x, y: percent.y },
+      undefined as never,
+      undefined as never,
+      vp,
+    );
+
+  test('truncates rather than rounds when the pixel is fractional', async () => {
+    // 1125 * 50 / 100 = 562.5 -> trunc 562 (rounding would give 563)
+    // 2436 * 33 / 100 = 803.88 -> trunc 803 (rounding would give 804)
+    const point = await resolve({ x: 50, y: 33 }, viewport(1125, 2436));
+    assert.deepEqual(point, { x: 562, y: 803 });
+  });
+
+  test('truncates fractions above .5, where rounding would go up', async () => {
+    // Both fractions round UP, so these only pass under truncation:
+    // 1179 * 5 / 100 = 58.95 -> trunc 58 (round -> 59)
+    // 2556 * 35 / 100 = 894.6 -> trunc 894 (round -> 895)
+    const point = await resolve({ x: 5, y: 35 }, viewport(1179, 2556));
+    assert.deepEqual(point, { x: 58, y: 894 });
+  });
+
+  test('is exact when the pixel is whole', async () => {
+    const point = await resolve({ x: 50, y: 25 }, viewport(1080, 1920));
+    assert.deepEqual(point, { x: 540, y: 480 });
+  });
+
+  test('adds the viewport origin after truncating the fraction', async () => {
+    // Origin is added to the truncated span, not truncated together with it.
+    const point = await resolve({ x: 50, y: 50 }, viewport(1125, 2436, 7, 11));
+    assert.deepEqual(point, { x: 7 + 562, y: 11 + 1218 });
+  });
+
+  test('0% and 100% map to the viewport edges', async () => {
+    assert.deepEqual(await resolve({ x: 0, y: 0 }, viewport(1125, 2436)), { x: 0, y: 0 });
+    assert.deepEqual(await resolve({ x: 100, y: 100 }, viewport(1125, 2436)), { x: 1125, y: 2436 });
+  });
+
+  test('absolute coordinates pass through untouched', async () => {
+    const point = await resolveMaestroCoordinate(
+      { space: 'absolute', x: 100, y: 200 },
+      undefined as never,
+      undefined as never,
+      viewport(1125, 2436),
+    );
+    assert.deepEqual(point, { x: 100, y: 200 });
+  });
+});