Extension Conformance #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Extension Conformance | |
| on: | |
| pull_request: | |
| schedule: | |
| # Nightly full official conformance (02:00 UTC) | |
| - cron: "0 2 * * *" | |
| # Weekly extended conformance (Saturday 02:00 UTC) | |
| - cron: "0 2 * * 6" | |
| workflow_dispatch: | |
| inputs: | |
| profile: | |
| description: "Conformance profile to run" | |
| required: false | |
| default: "auto" | |
| type: choice | |
| options: | |
| - auto | |
| - fast | |
| - full | |
| - weekly | |
| # Deterministic execution settings applied to all jobs. | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: "1" | |
| RUST_TEST_THREADS: "1" | |
| PI_TEST_MODE: "1" | |
| PI_TS_ORACLE_TIMEOUT_SECS: "30" | |
| PI_CONFORMANCE_SEED: "42" | |
| jobs: | |
| # ─── Shared setup (reusable via needs) ──────────────────────────────────── | |
| conformance-matrix: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: pi_agent_rust | |
| shell: bash | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # PR: fast conformance (official sample + generated tier 1-2 + negative) | |
| - name: fast-official | |
| if_event: pull_request | |
| test_cmd: cargo test --test ext_conformance_diff --features ext-conformance -- --nocapture | |
| env_extra: "PI_OFFICIAL_MAX=5" | |
| - name: fast-generated | |
| if_event: pull_request | |
| test_cmd: cargo test --test ext_conformance_generated --features ext-conformance -- --nocapture | |
| env_extra: "" | |
| - name: fast-negative | |
| if_event: pull_request | |
| test_cmd: cargo test --test extensions_policy_negative -- --nocapture | |
| env_extra: "" | |
| - name: fast-capability-matrix | |
| if_event: pull_request | |
| test_cmd: cargo test --test capability_denial_matrix -- --nocapture | |
| env_extra: "" | |
| # Nightly: full official + all tiers + scenarios + fixture schema + artifacts | |
| - name: full-official | |
| if_event: nightly | |
| test_cmd: cargo test --test ext_conformance_diff --features ext-conformance -- --nocapture | |
| env_extra: "" | |
| - name: full-generated | |
| if_event: nightly | |
| test_cmd: cargo test --test ext_conformance_generated --features ext-conformance -- --include-ignored --nocapture | |
| env_extra: "" | |
| - name: full-scenarios | |
| if_event: nightly | |
| test_cmd: cargo test --test ext_conformance_scenarios --features ext-conformance -- --nocapture | |
| env_extra: "" | |
| - name: full-negative | |
| if_event: nightly | |
| test_cmd: cargo test --test extensions_policy_negative -- --nocapture | |
| env_extra: "" | |
| - name: full-fixture-schema | |
| if_event: nightly | |
| test_cmd: cargo test --test ext_conformance_fixture_schema -- --nocapture | |
| env_extra: "" | |
| - name: full-artifacts | |
| if_event: nightly | |
| test_cmd: cargo test --test ext_conformance_artifacts -- --nocapture | |
| env_extra: "" | |
| - name: full-report | |
| if_event: nightly | |
| test_cmd: cargo test --test conformance_report generate_conformance_report -- --nocapture | |
| env_extra: "" | |
| # Weekly: community + npm + third-party | |
| - name: weekly-community | |
| if_event: weekly | |
| test_cmd: cargo test --test ext_conformance_diff --features ext-conformance -- --ignored --skip load_time_benchmark_official --skip event_dispatch_latency_benchmark | |
| env_extra: "" | |
| # Conditional: run matrix entries matching the current trigger. | |
| # PR -> fast-*, nightly schedule -> full-*, Saturday schedule -> weekly-*, | |
| # dispatch -> profile input. This cannot live in jobs.<job_id>.if because | |
| # GitHub evaluates job if conditions before matrix expansion. | |
| env: | |
| SHOULD_RUN: ${{ (github.event_name == 'pull_request' && contains(matrix.if_event, 'pull_request')) || (github.event_name == 'schedule' && github.event.schedule == '0 2 * * *' && contains(matrix.if_event, 'nightly')) || (github.event_name == 'schedule' && github.event.schedule == '0 2 * * 6' && contains(matrix.if_event, 'weekly')) || (github.event_name == 'workflow_dispatch' && (inputs.profile == 'auto' || (inputs.profile == 'fast' && contains(matrix.if_event, 'pull_request')) || (inputs.profile == 'full' && contains(matrix.if_event, 'nightly')) || (inputs.profile == 'weekly' && contains(matrix.if_event, 'weekly')))) }} | |
| name: "conformance (${{ matrix.name }})" | |
| steps: | |
| - name: Skip unmatched profile | |
| if: env.SHOULD_RUN != 'true' | |
| run: echo "Skipping ${{ matrix.name }} for event/profile selection" | |
| - name: Free disk space | |
| if: env.SHOULD_RUN == 'true' | |
| working-directory: / | |
| run: | | |
| set -euxo pipefail | |
| df -h / | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ | |
| /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go \ | |
| /opt/hostedtoolcache/Python /opt/hostedtoolcache/Ruby \ | |
| /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk \ | |
| /usr/local/share/powershell /usr/share/swift \ | |
| /usr/local/graalvm /usr/local/.ghcup /usr/local/julia* | |
| sudo docker image prune --all --force || true | |
| sudo apt-get clean || true | |
| df -h / | |
| - name: Checkout pi_agent_rust | |
| if: env.SHOULD_RUN == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pi_agent_rust | |
| - name: Checkout asupersync | |
| if: env.SHOULD_RUN == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Dicklesworthstone/asupersync | |
| path: asupersync | |
| fetch-depth: 1 | |
| - name: Checkout rich_rust | |
| if: env.SHOULD_RUN == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Dicklesworthstone/rich_rust | |
| path: rich_rust | |
| fetch-depth: 1 | |
| - name: Checkout charmed_rust | |
| if: env.SHOULD_RUN == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Dicklesworthstone/charmed_rust | |
| path: charmed_rust | |
| fetch-depth: 1 | |
| - name: Checkout sqlmodel_rust | |
| if: env.SHOULD_RUN == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Dicklesworthstone/sqlmodel_rust | |
| path: sqlmodel_rust | |
| fetch-depth: 1 | |
| - name: Install system deps (fd, rg, xcb) [linux] | |
| if: env.SHOULD_RUN == 'true' | |
| run: | | |
| set -euxo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y fd-find ripgrep libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev | |
| sudo ln -sf "$(command -v fdfind)" /usr/local/bin/fd | |
| - name: Install Rust toolchain (nightly) | |
| if: env.SHOULD_RUN == 'true' | |
| uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: rustfmt, clippy | |
| - name: Cargo cache | |
| if: env.SHOULD_RUN == 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| pi_agent_rust/target/ | |
| key: conformance-${{ runner.os }}-${{ hashFiles('pi_agent_rust/Cargo.lock') }} | |
| restore-keys: conformance-${{ runner.os }}- | |
| - name: Install Bun | |
| if: env.SHOULD_RUN == 'true' | |
| uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: "1.3.8" | |
| - name: Install pi-mono dependencies | |
| if: env.SHOULD_RUN == 'true' | |
| working-directory: pi_agent_rust/legacy_pi_mono_code/pi-mono | |
| run: npm ci | |
| - name: Symlink bun for conformance tests | |
| if: env.SHOULD_RUN == 'true' | |
| run: | | |
| set -euxo pipefail | |
| sudo mkdir -p /home/ubuntu/.bun/bin | |
| sudo ln -sf "$(command -v bun)" /home/ubuntu/.bun/bin/bun | |
| - name: "Run: ${{ matrix.name }}" | |
| if: env.SHOULD_RUN == 'true' | |
| env: | |
| PI_OFFICIAL_MAX: ${{ matrix.env_extra == 'PI_OFFICIAL_MAX=5' && '5' || '' }} | |
| PI_CONFORMANCE_MAX_RETRIES: ${{ contains(matrix.if_event, 'nightly') && '2' || '0' }} | |
| PI_CONFORMANCE_RETRY_DELAY: "10" | |
| run: | | |
| set -euxo pipefail | |
| if [[ "${PI_CONFORMANCE_MAX_RETRIES}" -gt 0 ]]; then | |
| bash scripts/ci_conformance_retry.sh "${{ matrix.name }}" \ | |
| ${{ matrix.test_cmd }} 2>&1 | tee "conformance-${{ matrix.name }}.log" | |
| else | |
| ${{ matrix.test_cmd }} 2>&1 | tee "conformance-${{ matrix.name }}.log" | |
| fi | |
| - name: Upload conformance logs | |
| if: always() && env.SHOULD_RUN == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-${{ matrix.name }}-${{ github.sha }} | |
| path: | | |
| pi_agent_rust/conformance-${{ matrix.name }}.log | |
| pi_agent_rust/flake_events.jsonl | |
| pi_agent_rust/tests/ext_conformance/reports/**/*.json | |
| pi_agent_rust/tests/ext_conformance/reports/**/*.jsonl | |
| pi_agent_rust/tests/ext_conformance/reports/**/*.md | |
| if-no-files-found: warn | |
| # ─── Trend report (nightly only) ────────────────────────────────────── | |
| trend-report: | |
| needs: [conformance-matrix] | |
| if: >- | |
| github.event_name == 'schedule' && github.event.schedule == '0 2 * * *' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download conformance artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: conformance-full-report-* | |
| merge-multiple: true | |
| - name: Ensure reports dir and summary exist | |
| run: | | |
| set -euxo pipefail | |
| mkdir -p tests/ext_conformance/reports | |
| # download-artifact strips the common pi_agent_rust/ prefix, so the | |
| # summary lands directly in tests/ext_conformance/reports/. | |
| # Verify it exists (it may also already be there from the checkout). | |
| if [ -f tests/ext_conformance/reports/conformance_summary.json ]; then | |
| echo "conformance_summary.json found" | |
| else | |
| echo "WARNING: conformance_summary.json not found after artifact download" | |
| ls -la tests/ext_conformance/reports/ || true | |
| fi | |
| - name: Generate trend entry and report | |
| env: | |
| GIT_SHA: ${{ github.sha }} | |
| GIT_REF: ${{ github.ref_name }} | |
| run: python3 scripts/conformance_trend.py | |
| - name: Upload trend report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-trend-${{ github.sha }} | |
| path: | | |
| tests/ext_conformance/reports/conformance_trend.jsonl | |
| tests/ext_conformance/reports/TREND_REPORT.md | |
| if-no-files-found: warn |