conformance-differential.yml

name: Conformance Differential

# Layer 3 of the Maestro conformance oracle: run a small set of flows through
# BOTH real Maestro and agent-device on a live device and compare app-observable
# outcomes, plus assert engine-side timing invariants over agent-device's own
# replay trace. This is the only layer that can catch settle-loop ordering, which
# has no reflectable upstream constant.
#
# Scheduled: this path is proven end-to-end (run 29504440599 executed both
# engines against the real fixture app). Scenarios that currently diverge are
# DECLARED via `knownDivergence` in differential/scenarios.ts with a tracking
# issue, so the schedule stays green on known gaps and fails only on undeclared
# ones — the same contract layer 1 has. A declaration that stops reproducing also
# fails, so a fix cannot land while leaving the oracle blind.

on:
  schedule:
    - cron: "0 5 * * *"
  workflow_dispatch:
    inputs:
      only:
        description: "Single scenario id to run (default: all)"
        required: false

permissions:
  contents: read
  # setup-fixture-app looks up the cached test-app binary via the artifacts API.
  actions: read

concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  AGENT_DEVICE_CLI: "--experimental-strip-types src/bin.ts"
  DIFFERENTIAL_ONLY: ${{ github.event.inputs.only || '' }}
  # CI should not phone home, and it keeps `maestro --version` to just the
  # version instead of prefixing an analytics notice.
  MAESTRO_CLI_NO_ANALYTICS: "1"

jobs:
  differential-ios:
    name: iOS Conformance Differential
    runs-on: macos-26
    timeout-minutes: 90
    env:
      IOS_RUNTIME_VERSION: "26.2"
      AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
      AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup toolchain
        uses: ./.github/actions/setup-node-pnpm

      - name: Setup Apple replay
        uses: ./.github/actions/setup-apple-replay
        with:
          derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
          cache-key-prefix: ios-runner-prebuilt
          cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
          build-command: sh ./scripts/build-xcuitest-apple.sh
          xcuitest-platform: ios

      - name: Boot simulator
        uses: ./.github/actions/boot-ios-test-simulator
        with:
          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
          preferred-device-name: iPhone 17 Pro

      # Installs the cached Release test-app and refreshes its JS. Shared with any
      # other job that needs a controlled app to drive (see #320) — the artifact
      # is repo-wide, so once the producer builds a fingerprint every workflow
      # gets it as a download.
      - name: Setup fixture app
        id: fixture-app
        uses: ./.github/actions/setup-fixture-app
        with:
          device-name: iPhone 17 Pro

      # The action guarantees *an* app is installed; this asserts it is the one
      # the scenarios actually target, so a fixture rename cannot leave the
      # differential driving the wrong app.
      - name: Verify the installed app is the one the scenarios target
        run: |
          set -euo pipefail
          EXPECTED=$(node --experimental-strip-types -e \
            "import('./scripts/maestro-conformance/differential/scenarios.ts').then(m => console.log(m.DIFFERENTIAL_APP_ID))")
          ACTUAL="${{ steps.fixture-app.outputs.app-id }}"
          echo "expected=$EXPECTED actual=$ACTUAL (source=${{ steps.fixture-app.outputs.source }})"
          if [ "$ACTUAL" != "$EXPECTED" ]; then
            echo "::error::Installed fixture app is $ACTUAL but the scenarios target $EXPECTED; they would fail vacuously."
            exit 1
          fi

      # Pin the Maestro CLI to the SAME version the oracle pins for layers 1-2,
      # read from pinned-upstream.json so the two can never drift apart. The
      # installer honours MAESTRO_VERSION; assert what actually landed.
      - name: Install pinned Maestro CLI
        run: |
          set -euo pipefail
          MAESTRO_VERSION=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
          export MAESTRO_VERSION
          echo "Pinning Maestro CLI to $MAESTRO_VERSION (from pinned-upstream.json)"
          curl -fsSL "https://get.maestro.mobile.dev" | bash
          echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"

      - name: Verify the Maestro CLI matches the oracle pin
        run: |
          set -euo pipefail
          EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
          # `maestro --version` can print notices (e.g. the analytics banner)
          # around the version, so match the semver line rather than the whole
          # output. Empty ACTUAL fails the comparison below, as it should.
          ACTUAL=$(maestro --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | tail -1 || true)
          echo "expected=$EXPECTED actual=$ACTUAL"
          if [ "$ACTUAL" != "$EXPECTED" ]; then
            echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior."
            exit 1
          fi

      # --trace-root lets the runner find agent-device's replay-timing.ndjson and
      # assert engine-side invariants (e.g. the settle loop latches instead of
      # burning its budget) — outcome parity alone cannot see that.
      - name: Run differential
        run: |
          node --experimental-strip-types scripts/maestro-conformance/differential/run.ts \
            --platform ios \
            --out-dir "${{ github.workspace }}/.tmp/conformance-differential" \
            --trace-root "${{ github.workspace }}/.agent-device" \
            ${DIFFERENTIAL_ONLY:+--only "$DIFFERENTIAL_ONLY"}

      # Keep the trace, not just the verdict: the engine-side invariants are
      # computed FROM replay-timing.ndjson, so without it a report saying
      # "invariant violated: tapRetries was 0" cannot be audited or debugged
      # after the runner is gone.
      - name: Upload report and trace
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: conformance-differential-ios
          path: |
            .tmp/conformance-differential/differential-report.json
            .agent-device/**/replay-timing.ndjson
          if-no-files-found: ignore