conformance.yml

name: Extension Conformance

on:
  pull_request:
  schedule:
    # Nightly full official conformance (02:00 UTC)
    - cron: "0 2 * * *"
    # Weekly extended conformance (Saturday 02:00 UTC)
    - cron: "0 2 * * 6"
  workflow_dispatch:
    inputs:
      profile:
        description: "Conformance profile to run"
        required: false
        default: "auto"
        type: choice
        options:
          - auto
          - fast
          - full
          - weekly

# Deterministic execution settings applied to all jobs.
env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: "1"
  RUST_TEST_THREADS: "1"
  PI_TEST_MODE: "1"
  PI_TS_ORACLE_TIMEOUT_SECS: "30"
  PI_CONFORMANCE_SEED: "42"

jobs:
  # ─── Shared setup (reusable via needs) ────────────────────────────────────

  conformance-matrix:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: pi_agent_rust
        shell: bash

    strategy:
      fail-fast: false
      matrix:
        include:
          # PR: fast conformance (official sample + generated tier 1-2 + negative)
          - name: fast-official
            if_event: pull_request
            test_cmd: cargo test --test ext_conformance_diff --features ext-conformance -- --nocapture
            env_extra: "PI_OFFICIAL_MAX=5"
          - name: fast-generated
            if_event: pull_request
            test_cmd: cargo test --test ext_conformance_generated --features ext-conformance -- --nocapture
            env_extra: ""
          - name: fast-negative
            if_event: pull_request
            test_cmd: cargo test --test extensions_policy_negative -- --nocapture
            env_extra: ""
          - name: fast-capability-matrix
            if_event: pull_request
            test_cmd: cargo test --test capability_denial_matrix -- --nocapture
            env_extra: ""
          # Nightly: full official + all tiers + scenarios + fixture schema + artifacts
          - name: full-official
            if_event: nightly
            test_cmd: cargo test --test ext_conformance_diff --features ext-conformance -- --nocapture
            env_extra: ""
          - name: full-generated
            if_event: nightly
            test_cmd: cargo test --test ext_conformance_generated --features ext-conformance -- --include-ignored --nocapture
            env_extra: ""
          - name: full-scenarios
            if_event: nightly
            test_cmd: cargo test --test ext_conformance_scenarios --features ext-conformance -- --nocapture
            env_extra: ""
          - name: full-negative
            if_event: nightly
            test_cmd: cargo test --test extensions_policy_negative -- --nocapture
            env_extra: ""
          - name: full-fixture-schema
            if_event: nightly
            test_cmd: cargo test --test ext_conformance_fixture_schema -- --nocapture
            env_extra: ""
          - name: full-artifacts
            if_event: nightly
            test_cmd: cargo test --test ext_conformance_artifacts -- --nocapture
            env_extra: ""
          - name: full-report
            if_event: nightly
            test_cmd: cargo test --test conformance_report generate_conformance_report -- --nocapture
            env_extra: ""
          # Weekly: community + npm + third-party
          - name: weekly-community
            if_event: weekly
            test_cmd: cargo test --test ext_conformance_diff --features ext-conformance -- --ignored --skip load_time_benchmark_official --skip event_dispatch_latency_benchmark
            env_extra: ""

    # Conditional: run matrix entries matching the current trigger.
    # PR -> fast-*, nightly schedule -> full-*, Saturday schedule -> weekly-*,
    # dispatch -> profile input. This cannot live in jobs.<job_id>.if because
    # GitHub evaluates job if conditions before matrix expansion.
    env:
      SHOULD_RUN: ${{ (github.event_name == 'pull_request' && contains(matrix.if_event, 'pull_request')) || (github.event_name == 'schedule' && github.event.schedule == '0 2 * * *' && contains(matrix.if_event, 'nightly')) || (github.event_name == 'schedule' && github.event.schedule == '0 2 * * 6' && contains(matrix.if_event, 'weekly')) || (github.event_name == 'workflow_dispatch' && (inputs.profile == 'auto' || (inputs.profile == 'fast' && contains(matrix.if_event, 'pull_request')) || (inputs.profile == 'full' && contains(matrix.if_event, 'nightly')) || (inputs.profile == 'weekly' && contains(matrix.if_event, 'weekly')))) }}

    name: "conformance (${{ matrix.name }})"

    steps:
      - name: Skip unmatched profile
        if: env.SHOULD_RUN != 'true'
        run: echo "Skipping ${{ matrix.name }} for event/profile selection"

      - name: Free disk space
        if: env.SHOULD_RUN == 'true'
        working-directory: /
        run: |
          set -euxo pipefail
          df -h /
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go \
            /opt/hostedtoolcache/Python /opt/hostedtoolcache/Ruby \
            /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk \
            /usr/local/share/powershell /usr/share/swift \
            /usr/local/graalvm /usr/local/.ghcup /usr/local/julia*
          sudo docker image prune --all --force || true
          sudo apt-get clean || true
          df -h /

      - name: Checkout pi_agent_rust
        if: env.SHOULD_RUN == 'true'
        uses: actions/checkout@v4
        with:
          path: pi_agent_rust

      - name: Checkout asupersync
        if: env.SHOULD_RUN == 'true'
        uses: actions/checkout@v4
        with:
          repository: Dicklesworthstone/asupersync
          path: asupersync
          fetch-depth: 1

      - name: Checkout rich_rust
        if: env.SHOULD_RUN == 'true'
        uses: actions/checkout@v4
        with:
          repository: Dicklesworthstone/rich_rust
          path: rich_rust
          fetch-depth: 1

      - name: Checkout charmed_rust
        if: env.SHOULD_RUN == 'true'
        uses: actions/checkout@v4
        with:
          repository: Dicklesworthstone/charmed_rust
          path: charmed_rust
          fetch-depth: 1

      - name: Checkout sqlmodel_rust
        if: env.SHOULD_RUN == 'true'
        uses: actions/checkout@v4
        with:
          repository: Dicklesworthstone/sqlmodel_rust
          path: sqlmodel_rust
          fetch-depth: 1

      - name: Install system deps (fd, rg, xcb) [linux]
        if: env.SHOULD_RUN == 'true'
        run: |
          set -euxo pipefail
          sudo apt-get update
          sudo apt-get install -y fd-find ripgrep libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev
          sudo ln -sf "$(command -v fdfind)" /usr/local/bin/fd

      - name: Install Rust toolchain (nightly)
        if: env.SHOULD_RUN == 'true'
        uses: dtolnay/rust-toolchain@nightly
        with:
          components: rustfmt, clippy

      - name: Cargo cache
        if: env.SHOULD_RUN == 'true'
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            pi_agent_rust/target/
          key: conformance-${{ runner.os }}-${{ hashFiles('pi_agent_rust/Cargo.lock') }}
          restore-keys: conformance-${{ runner.os }}-

      - name: Install Bun
        if: env.SHOULD_RUN == 'true'
        uses: oven-sh/setup-bun@v1
        with:
          bun-version: "1.3.8"

      - name: Install pi-mono dependencies
        if: env.SHOULD_RUN == 'true'
        working-directory: pi_agent_rust/legacy_pi_mono_code/pi-mono
        run: npm ci

      - name: Symlink bun for conformance tests
        if: env.SHOULD_RUN == 'true'
        run: |
          set -euxo pipefail
          sudo mkdir -p /home/ubuntu/.bun/bin
          sudo ln -sf "$(command -v bun)" /home/ubuntu/.bun/bin/bun

      - name: "Run: ${{ matrix.name }}"
        if: env.SHOULD_RUN == 'true'
        env:
          PI_OFFICIAL_MAX: ${{ matrix.env_extra == 'PI_OFFICIAL_MAX=5' && '5' || '' }}
          PI_CONFORMANCE_MAX_RETRIES: ${{ contains(matrix.if_event, 'nightly') && '2' || '0' }}
          PI_CONFORMANCE_RETRY_DELAY: "10"
        run: |
          set -euxo pipefail
          if [[ "${PI_CONFORMANCE_MAX_RETRIES}" -gt 0 ]]; then
            bash scripts/ci_conformance_retry.sh "${{ matrix.name }}" \
              ${{ matrix.test_cmd }} 2>&1 | tee "conformance-${{ matrix.name }}.log"
          else
            ${{ matrix.test_cmd }} 2>&1 | tee "conformance-${{ matrix.name }}.log"
          fi

      - name: Upload conformance logs
        if: always() && env.SHOULD_RUN == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: conformance-${{ matrix.name }}-${{ github.sha }}
          path: |
            pi_agent_rust/conformance-${{ matrix.name }}.log
            pi_agent_rust/flake_events.jsonl
            pi_agent_rust/tests/ext_conformance/reports/**/*.json
            pi_agent_rust/tests/ext_conformance/reports/**/*.jsonl
            pi_agent_rust/tests/ext_conformance/reports/**/*.md
          if-no-files-found: warn

  # ─── Trend report (nightly only) ──────────────────────────────────────
  trend-report:
    needs: [conformance-matrix]
    if: >-
      github.event_name == 'schedule' && github.event.schedule == '0 2 * * *'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download conformance artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: conformance-full-report-*
          merge-multiple: true

      - name: Ensure reports dir and summary exist
        run: |
          set -euxo pipefail
          mkdir -p tests/ext_conformance/reports
          # download-artifact strips the common pi_agent_rust/ prefix, so the
          # summary lands directly in tests/ext_conformance/reports/.
          # Verify it exists (it may also already be there from the checkout).
          if [ -f tests/ext_conformance/reports/conformance_summary.json ]; then
            echo "conformance_summary.json found"
          else
            echo "WARNING: conformance_summary.json not found after artifact download"
            ls -la tests/ext_conformance/reports/ || true
          fi

      - name: Generate trend entry and report
        env:
          GIT_SHA: ${{ github.sha }}
          GIT_REF: ${{ github.ref_name }}
        run: python3 scripts/conformance_trend.py

      - name: Upload trend report
        uses: actions/upload-artifact@v4
        with:
          name: conformance-trend-${{ github.sha }}
          path: |
            tests/ext_conformance/reports/conformance_trend.jsonl
            tests/ext_conformance/reports/TREND_REPORT.md
          if-no-files-found: warn