Conformance Differential #14
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Conformance Differential | |
| # Layer 3 of the Maestro conformance oracle: run a small set of flows through | |
| # BOTH real Maestro and agent-device on a live device and compare app-observable | |
| # outcomes, plus assert engine-side timing invariants over agent-device's own | |
| # replay trace. This is the only layer that can catch settle-loop ordering, which | |
| # has no reflectable upstream constant. | |
| # | |
| # Scheduled: this path is proven end-to-end (run 29504440599 executed both | |
| # engines against the real fixture app). Scenarios that currently diverge are | |
| # DECLARED via `knownDivergence` in differential/scenarios.ts with a tracking | |
| # issue, so the schedule stays green on known gaps and fails only on undeclared | |
| # ones — the same contract layer 1 has. A declaration that stops reproducing also | |
| # fails, so a fix cannot land while leaving the oracle blind. | |
| on: | |
| schedule: | |
| - cron: "0 5 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| only: | |
| description: "Single scenario id to run (default: all)" | |
| required: false | |
| permissions: | |
| contents: read | |
| # setup-fixture-app looks up the cached test-app binary via the artifacts API. | |
| actions: read | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| AGENT_DEVICE_CLI: "--experimental-strip-types src/bin.ts" | |
| DIFFERENTIAL_ONLY: ${{ github.event.inputs.only || '' }} | |
| # CI should not phone home, and it keeps `maestro --version` to just the | |
| # version instead of prefixing an analytics notice. | |
| MAESTRO_CLI_NO_ANALYTICS: "1" | |
| jobs: | |
| differential-ios: | |
| name: iOS Conformance Differential | |
| runs-on: macos-26 | |
| timeout-minutes: 90 | |
| env: | |
| IOS_RUNTIME_VERSION: "26.2" | |
| AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state | |
| AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup toolchain | |
| uses: ./.github/actions/setup-node-pnpm | |
| - name: Setup Apple replay | |
| uses: ./.github/actions/setup-apple-replay | |
| with: | |
| derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} | |
| cache-key-prefix: ios-runner-prebuilt | |
| cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} | |
| build-command: sh ./scripts/build-xcuitest-apple.sh | |
| xcuitest-platform: ios | |
| - name: Boot simulator | |
| uses: ./.github/actions/boot-ios-test-simulator | |
| with: | |
| runtime-version: ${{ env.IOS_RUNTIME_VERSION }} | |
| preferred-device-name: iPhone 17 Pro | |
| # Installs the cached Release test-app and refreshes its JS. Shared with any | |
| # other job that needs a controlled app to drive (see #320) — the artifact | |
| # is repo-wide, so once the producer builds a fingerprint every workflow | |
| # gets it as a download. | |
| - name: Setup fixture app | |
| id: fixture-app | |
| uses: ./.github/actions/setup-fixture-app | |
| with: | |
| device-name: iPhone 17 Pro | |
| # The action guarantees *an* app is installed; this asserts it is the one | |
| # the scenarios actually target, so a fixture rename cannot leave the | |
| # differential driving the wrong app. | |
| - name: Verify the installed app is the one the scenarios target | |
| run: | | |
| set -euo pipefail | |
| EXPECTED=$(node --experimental-strip-types -e \ | |
| "import('./scripts/maestro-conformance/differential/scenarios.ts').then(m => console.log(m.DIFFERENTIAL_APP_ID))") | |
| ACTUAL="${{ steps.fixture-app.outputs.app-id }}" | |
| echo "expected=$EXPECTED actual=$ACTUAL (source=${{ steps.fixture-app.outputs.source }})" | |
| if [ "$ACTUAL" != "$EXPECTED" ]; then | |
| echo "::error::Installed fixture app is $ACTUAL but the scenarios target $EXPECTED; they would fail vacuously." | |
| exit 1 | |
| fi | |
| # Pin the Maestro CLI to the SAME version the oracle pins for layers 1-2, | |
| # read from pinned-upstream.json so the two can never drift apart. The | |
| # installer honours MAESTRO_VERSION; assert what actually landed. | |
| - name: Install pinned Maestro CLI | |
| run: | | |
| set -euo pipefail | |
| MAESTRO_VERSION=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version") | |
| export MAESTRO_VERSION | |
| echo "Pinning Maestro CLI to $MAESTRO_VERSION (from pinned-upstream.json)" | |
| curl -fsSL "https://get.maestro.mobile.dev" | bash | |
| echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" | |
| - name: Verify the Maestro CLI matches the oracle pin | |
| run: | | |
| set -euo pipefail | |
| EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version") | |
| # `maestro --version` can print notices (e.g. the analytics banner) | |
| # around the version, so match the semver line rather than the whole | |
| # output. Empty ACTUAL fails the comparison below, as it should. | |
| ACTUAL=$(maestro --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | tail -1 || true) | |
| echo "expected=$EXPECTED actual=$ACTUAL" | |
| if [ "$ACTUAL" != "$EXPECTED" ]; then | |
| echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior." | |
| exit 1 | |
| fi | |
| # --trace-root lets the runner find agent-device's replay-timing.ndjson and | |
| # assert engine-side invariants (e.g. the settle loop latches instead of | |
| # burning its budget) — outcome parity alone cannot see that. | |
| - name: Run differential | |
| run: | | |
| node --experimental-strip-types scripts/maestro-conformance/differential/run.ts \ | |
| --platform ios \ | |
| --out-dir "${{ github.workspace }}/.tmp/conformance-differential" \ | |
| --trace-root "${{ github.workspace }}/.agent-device" \ | |
| ${DIFFERENTIAL_ONLY:+--only "$DIFFERENTIAL_ONLY"} | |
| # Keep the trace, not just the verdict: the engine-side invariants are | |
| # computed FROM replay-timing.ndjson, so without it a report saying | |
| # "invariant violated: tapRetries was 0" cannot be audited or debugged | |
| # after the runner is gone. | |
| - name: Upload report and trace | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: conformance-differential-ios | |
| path: | | |
| .tmp/conformance-differential/differential-report.json | |
| .agent-device/**/replay-timing.ndjson | |
| if-no-files-found: ignore |