conformance-differential.yml

name: Conformance Differential

# Layer 3 of the Maestro conformance oracle: run a small set of flows through
# BOTH real Maestro and agent-device on a live device and compare app-observable
# outcomes, plus assert engine-side timing invariants over agent-device's own
# replay trace. This is the only layer that can catch settle-loop ordering, which
# has no reflectable upstream constant.
#
# DISPATCH-ONLY, DELIBERATELY. This path has never executed end-to-end: nothing
# else in this repo builds or installs the Expo fixture app, so the build/install
# steps below are new and unproven. A nightly cron that fails at 05:00 every day
# teaches us nothing and trains people to ignore it. Run it manually, watch it,
# fix what breaks, and only then add:
#   schedule:
#     - cron: "0 5 * * *"
# Tracked as the follow-up to #1274; layers 1-2 verify per-PR without any device.

on:
  workflow_dispatch:
    inputs:
      only:
        description: "Single scenario id to run (default: all)"
        required: false
  # TEMPORARY — REMOVE BEFORE MERGE.
  # workflow_dispatch only registers once a workflow is on the default branch, so
  # this path cannot be dispatched pre-merge (the API returns 404). This trigger
  # exists solely to prove the never-executed device path on the PR head and
  # capture evidence. Delete this `push:` block once that run is green.
  push:
    branches:
      - claude/maestro-conformance-oracle-acf8d5
    paths:
      - "scripts/maestro-conformance/**"
      - ".github/workflows/conformance-differential.yml"

permissions:
  contents: read

concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  AGENT_DEVICE_CLI: "--experimental-strip-types src/bin.ts"
  DIFFERENTIAL_ONLY: ${{ github.event.inputs.only || '' }}

jobs:
  differential-ios:
    name: iOS Conformance Differential
    runs-on: macos-26
    timeout-minutes: 90
    env:
      IOS_RUNTIME_VERSION: "26.2"
      AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
      AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup toolchain
        uses: ./.github/actions/setup-node-pnpm

      - name: Setup Apple replay
        uses: ./.github/actions/setup-apple-replay
        with:
          derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
          cache-key-prefix: ios-runner-prebuilt
          cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
          build-command: sh ./scripts/build-xcuitest-apple.sh
          xcuitest-platform: ios

      - name: Boot simulator
        uses: ./.github/actions/boot-ios-test-simulator
        with:
          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
          preferred-device-name: iPhone 17 Pro

      # The scenarios drive the real fixture app (com.callstack.agentdevicelab).
      # Release avoids needing a Metro server for the JS bundle; `expo run:ios`
      # prebuilds, builds, installs onto the booted simulator, and launches.
      - name: Build and install the fixture app
        run: |
          set -euo pipefail
          pnpm test-app:install
          pnpm --dir examples/test-app exec expo run:ios \
            --configuration Release \
            --device "iPhone 17 Pro" \
            --no-bundler

      - name: Verify the fixture app is installed
        run: |
          set -euo pipefail
          # Single source of truth: the same constant the scenarios assert against.
          APP_ID=$(node --experimental-strip-types -e \
            "import('./scripts/maestro-conformance/differential/scenarios.ts').then(m => console.log(m.DIFFERENTIAL_APP_ID))")
          if ! xcrun simctl get_app_container booted "$APP_ID" >/dev/null 2>&1; then
            echo "::error::Fixture app $APP_ID is not installed on the booted simulator; layer-3 scenarios would fail vacuously."
            exit 1
          fi
          echo "Fixture app $APP_ID installed."

      # Pin the Maestro CLI to the SAME version the oracle pins for layers 1-2,
      # read from pinned-upstream.json so the two can never drift apart. The
      # installer honours MAESTRO_VERSION; assert what actually landed.
      - name: Install pinned Maestro CLI
        run: |
          set -euo pipefail
          MAESTRO_VERSION=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
          export MAESTRO_VERSION
          echo "Pinning Maestro CLI to $MAESTRO_VERSION (from pinned-upstream.json)"
          curl -fsSL "https://get.maestro.mobile.dev" | bash
          echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"

      - name: Verify the Maestro CLI matches the oracle pin
        run: |
          set -euo pipefail
          EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
          ACTUAL=$(maestro --version 2>/dev/null | tr -d '[:space:]')
          echo "expected=$EXPECTED actual=$ACTUAL"
          if [ "$ACTUAL" != "$EXPECTED" ]; then
            echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior."
            exit 1
          fi

      # --trace-root lets the runner find agent-device's replay-timing.ndjson and
      # assert engine-side invariants (e.g. the settle loop latches instead of
      # burning its budget) — outcome parity alone cannot see that.
      - name: Run differential
        run: |
          node --experimental-strip-types scripts/maestro-conformance/differential/run.ts \
            --platform ios \
            --out-dir "${{ github.workspace }}/.tmp/conformance-differential" \
            --trace-root "${{ github.workspace }}/.agent-device" \
            ${DIFFERENTIAL_ONLY:+--only "$DIFFERENTIAL_ONLY"}

      - name: Upload report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: conformance-differential-ios
          path: .tmp/conformance-differential/differential-report.json
          if-no-files-found: ignore