ci.yml

# SPDX-License-Identifier: AGPL-3.0-or-later
name: CI

on:
  push:
    branches: [ "main", "dev" ]
    paths-ignore:
      - '.agent/tracker/.ops/**'
      - '.agent/tracker-workspace/.ops/**'
  pull_request:
    branches: [ "main", "dev" ]
    paths-ignore:
      - '.agent/tracker/.ops/**'
      - '.agent/tracker-workspace/.ops/**'

concurrency:
  group: ci-${{ github.head_ref || github.ref_name }}
  cancel-in-progress: true

jobs:
  # Detect what files changed to skip expensive jobs for docs-only PRs
  # Skip entirely for tracker-sync PRs and tracker-only pushes
  # (handled by tracker-ci.yml — no need to run lint, pytest, etc.)
  changes:
    if: >-
      !startsWith(github.head_ref || '', 'tracker-sync/')
      && !startsWith(github.event.head_commit.message || '', 'tracker:')
    runs-on: self-hosted
    outputs:
      code: ${{ steps.filter.outputs.code }}
      tracker_data: ${{ steps.filter.outputs.tracker_data }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Detect code changes
        id: filter
        run: |
          # For PRs, compare against base branch
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            base="${{ github.event.pull_request.base.sha }}"
            head="${{ github.event.pull_request.head.sha }}"
          else
            # For push, compare against previous commit
            base="${{ github.event.before }}"
            head="${{ github.sha }}"
          fi

          # Handle initial push or empty/invalid base
          # github.event.before can be:
          # - All zeros: first push to branch
          # - Empty: some merge scenarios
          # - Invalid: ref no longer exists
          if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
            echo "code=true" >> "$GITHUB_OUTPUT"
            echo "Initial push or empty base - assuming code changes"
            exit 0
          fi

          # Verify base commit exists (handles invalid/missing refs)
          if ! git cat-file -e "$base" 2>/dev/null; then
            echo "code=true" >> "$GITHUB_OUTPUT"
            echo "Base commit $base not found - assuming code changes"
            exit 0
          fi

          # Check if any non-doc files changed
          # Docs: *.md, docs/**, LICENSE, .gitignore, etc.
          # Note: YAML files in .agent/tracker(-workspace)/ are tracker data,
          # not code — they should not trigger expensive test jobs.
          CODE_PATTERNS='\.py$|\.json$|\.toml$|pyproject\.toml|scripts/|\.github/workflows/'
          YAML_PATTERN='\.ya?ml$'
          TRACKER_DATA='^\.agent/tracker(-workspace)?/'

          CHANGED=$(git diff --name-only "$base" "$head")
          has_code=false
          echo "$CHANGED" | grep -qE "$CODE_PATTERNS" && has_code=true
          # YAML files count as code UNLESS they're tracker data
          echo "$CHANGED" | grep -E "$YAML_PATTERN" | grep -vqE "$TRACKER_DATA" 2>/dev/null && has_code=true
          echo "code=$has_code" >> "$GITHUB_OUTPUT"

          # Detect tracker data changes (for lightweight validation job)
          if echo "$CHANGED" | grep -qE '^\.agent/tracker(-workspace)?/|^packages/hypergumbo-tracker/'; then
            echo "tracker_data=true" >> "$GITHUB_OUTPUT"
            echo "Tracker data changes detected"
          else
            echo "tracker_data=false" >> "$GITHUB_OUTPUT"
          fi

          if [ "$has_code" = "true" ]; then
            echo "Code changes detected"
          else
            echo "Docs-only changes, skipping expensive jobs"
          fi

  # Stop-the-line protocol: Block PRs when full-suite is broken
  # Exception: PRs titled "fix(job-XXXXX):" are allowed through
  stop-the-line:
    runs-on: self-hosted
    if: github.event_name == 'pull_request'
    steps:
      - name: Check full-suite status
        run: |
          # Use commit status API (works unauthenticated for public repos)
          # This is more reliable than the workflow runs API on Forgejo
          API_BASE="https://codeberg.org/api/v1/repos/${{ github.repository }}"

          # Get the base branch HEAD SHA (the branch we're merging into)
          BASE_SHA="${{ github.event.pull_request.base.sha }}"
          echo "Checking commit status for base branch HEAD: $BASE_SHA"

          # Fetch commit status (no auth needed for public repos)
          STATUS_JSON=$(curl -s "$API_BASE/commits/$BASE_SHA/status" 2>/dev/null || echo '{"state":"unknown","statuses":[]}')

          # Parse the response — check only the aggregate job, which is the
          # authoritative verdict. Individual jobs like test-core may fail due to
          # Codeberg runner timeouts while their retries (on self-hosted) succeed.
          # The aggregate job accounts for retries, so it's the single source of truth.
          RESULT=$(echo "$STATUS_JSON" | python3 -c '
          import json, sys, re
          try:
              d = json.load(sys.stdin)
              statuses = d.get("statuses", [])

              # Find the aggregate job status specifically — it is the authoritative
              # verdict that accounts for retries (test-core-retry, etc.)
              aggregate = [s for s in statuses if "Full Test Suite / aggregate" in s.get("context", "")]

              if not aggregate:
                  # No aggregate job found — check if any Full Test Suite statuses exist at all
                  full_suite = [s for s in statuses if "Full Test Suite" in s.get("context", "")]
                  if not full_suite:
                      print("none|unknown")
                  else:
                      # Full suite is still running (aggregate has not reported yet)
                      print("none|pending")
                  sys.exit(0)

              agg = aggregate[0]
              agg_status = agg.get("status", "unknown")

              if agg_status == "failure":
                  # Extract run ID from target_url (e.g., /iterabloom/hypergumbo/actions/runs/1702/jobs/2)
                  url = agg.get("target_url", "")
                  match = re.search(r"/runs/(\d+)/", url)
                  run_id = match.group(1) if match else "unknown"
                  print(f"{run_id}|failure")
              elif agg_status == "success":
                  print("ok|success")
              else:
                  # pending or other — treat as no result yet
                  print("none|pending")
          except Exception as e:
              print(f"error|{e}")
          ' 2>/dev/null || echo "error|parse_failed")

          RUN_ID=$(echo "$RESULT" | cut -d'|' -f1)
          STATUS=$(echo "$RESULT" | cut -d'|' -f2)

          echo "Full Test Suite status: $STATUS (run: $RUN_ID)"

          if [[ "$STATUS" == "failure" ]]; then
            PR_TITLE="${{ github.event.pull_request.title }}"
            REQUIRED_PREFIX="fix(job-$RUN_ID):"

            echo "⚠️ Full suite is broken (run $RUN_ID failed)"

            if [[ "$PR_TITLE" == "$REQUIRED_PREFIX"* ]] || [[ "$PR_TITLE" == fix\(job-* ]]; then
              echo "✅ PR title indicates this is a fix - allowing through"
              echo "   Title: $PR_TITLE"
            else
              echo "❌ PRs blocked until full suite is fixed"
              echo ""
              echo "To unblock, either:"
              echo "  1. Fix the broken full suite"
              echo "  2. Rename your PR to start with: $REQUIRED_PREFIX"
              echo ""
              echo "Current PR title: $PR_TITLE"
              exit 1
            fi
          elif [[ "$STATUS" == "none" ]] || [[ "$STATUS" == "unknown" ]] || [[ "$STATUS" == "pending" ]]; then
            echo "⚠️ No Full Test Suite aggregate result found - allowing PR"
            echo "   (first run, still running, or aggregate not yet reported)"
          else
            echo "✅ Full suite is passing - PR allowed"
          fi

  lint:
    needs: [changes, stop-the-line]
    if: always() && needs.changes.outputs.code == 'true' && needs.stop-the-line.result != 'failure'
    runs-on: self-hosted
    env:
      SEGMENT_DOWNLOAD_TIMEOUT_MINS: "1"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          # No cache: 'pip' — setup-python@v5 bundles @actions/cache npm v4
          # which speaks Twirp v2 protocol. Forgejo's cache proxy only implements
          # v1 REST, so the v4 client hangs with ETIMEDOUT (~3 min per job).
      - name: Install linting tools
        run: |
          pip install --upgrade pip
          pip install ruff bandit
      - name: Run Ruff (linting)
        run: ruff check packages/*/src/ packages/*/tests/
      - name: Run Bandit (security)
        run: bandit -r packages/*/src/ -c pyproject.toml


  # Run bash hook tests on every PR regardless of what changed.
  # test_hooks.sh exercises pre-push and stop hook behavior in a sandbox;
  # it takes ~10 seconds and its failure mode is "silent rot" (the script
  # was manually-run only, so regressions could hide for months). Running
  # always is cheap and prevents that. See INV-pofam for history.
  hook-tests:
    needs: [stop-the-line]
    if: always() && needs.stop-the-line.result != 'failure'
    runs-on: self-hosted
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v4
      - name: Run .githooks/test_hooks.sh
        run: bash .githooks/test_hooks.sh


  dco:
    runs-on: self-hosted
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Check DCO sign-off
        run: |
          base="${{ github.event.pull_request.base.sha }}"
          head="${{ github.event.pull_request.head.sha }}"
          # Skip merge commits (--no-merges) since they're auto-generated by
          # the forge UI and don't need sign-off
          for sha in $(git rev-list --no-merges $base..$head); do
            if ! git log -1 --format='%B' "$sha" | grep -q '^Signed-off-by:'; then
              echo "❌ Commit $sha is missing DCO sign-off"
              echo "   Add with: git commit --amend -s"
              exit 1
            fi
          done
          echo "✅ All commits have DCO sign-off"

  audit:
    needs: [changes, stop-the-line]
    if: always() && needs.changes.outputs.code == 'true' && needs.stop-the-line.result != 'failure'
    runs-on: self-hosted
    env:
      SEGMENT_DOWNLOAD_TIMEOUT_MINS: "1"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install pip-audit
        run: |
          pip install --upgrade pip wheel setuptools
          pip install pip-audit
          # CVE-2026-25645: requests 2.32.5 — fix in 2.33.0
          pip install "requests~=2.33.0"
          # CVE-2026-4539: pygments AdlLexer ReDoS — fix in 2.20.0.
          # The self-hosted runner's baseline Python can ship an older
          # pygments. Mirror our pyproject.toml pin here so the audit scans
          # the same version our packages install at runtime.
          pip install "pygments~=2.20.0"
      - name: Run pip-audit (dependency vulnerabilities)
        run: |
          # CVE-2025-71176: pytest /tmp/pytest-of-{user} TOCTOU on UNIX.
          # Fix is pytest 9.0.3, but we are pinned below 9.0.0 transitively by
          # syrupy 4.8.0 (hard-pinned by pytest-textual-snapshot 1.1.0).
          # Impact assessment: local attack boundary only, reachable solely in
          # CI/dev environments — pytest is a dev-only dep, never installed on
          # end-user systems running hypergumbo. Our self-hosted runners are
          # single-tenant, so practical exploitability is ~0.
          # Exit plan: upstream PR to relax the syrupy pin is tracked at
          # https://github.com/Textualize/pytest-textual-snapshot/pull/24
          # When that merges and a new pytest-textual-snapshot release ships,
          # bump pytest to 9.0.3+ and drop this ignore.
          #
          # CVE-2026-3219: pip concatenated-ZIP+tar archive confusion.
          # CVSS 4.6 MEDIUM; AV:L, UI:A, VI:L only. Exploitation requires a
          # user to install a hand-crafted archive that is simultaneously a
          # valid ZIP and a valid tar — our CI only installs pinned deps
          # from PyPI, so the attack surface is zero on the self-hosted
          # runner. Fix is pip 26.1 (PR pypa/pip#13870) but that release
          # was not yet on PyPI when this ignore was added (CVE published
          # 2026-04-20); the existing `pip install --upgrade pip` step at
          # install time will pick it up automatically once it ships.
          # Exit plan: drop this ignore once pip-audit stops flagging pip.
          pip-audit --skip-editable --ignore-vuln CVE-2025-71176 --ignore-vuln CVE-2026-3219

  verify-generated:
    needs: [changes, stop-the-line]
    if: always() && needs.changes.outputs.code == 'true' && needs.stop-the-line.result != 'failure'
    runs-on: self-hosted
    env:
      SEGMENT_DOWNLOAD_TIMEOUT_MINS: "1"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: |
          pip install --upgrade pip
          # Install from packages if available, otherwise legacy
          if [ -d "packages/hypergumbo-core" ]; then
            # Install test deps directly - NEVER use meta-package (it pulls PyTorch/CUDA)
            pip install pytest pytest-cov pytest-xdist ruff bandit pip-audit jsonschema yamllint
            pip install -e packages/hypergumbo-core -e packages/hypergumbo-lang-mainstream -e packages/hypergumbo-lang-common -e packages/hypergumbo-lang-extended1 -e packages/hypergumbo-lang-rust-analyzer -e "packages/hypergumbo-tracker[dev]"
          else
            pip install -e .[dev]
          fi
      - name: Verify schema.json is up-to-date
        run: ./scripts/generate-schema --check
      - name: Verify ARCHITECTURE.md is up-to-date
        run: ./scripts/generate-architecture --check

  # Build grammars once, cache wheels for pytest job (saves ~1 minute)
  build-grammars:
    needs: [changes, stop-the-line]
    if: always() && needs.changes.outputs.code == 'true' && needs.stop-the-line.result != 'failure'
    runs-on: self-hosted
    env:
      SEGMENT_DOWNLOAD_TIMEOUT_MINS: "1"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Cache grammar wheels
        id: cache
        uses: actions/cache@v3
        with:
          path: /tmp/grammar-wheels
          key: grammar-wheels-${{ hashFiles('scripts/build-source-grammars') }}
      - name: Build grammar wheels
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python3 -m pip install --upgrade pip setuptools wheel
          ./scripts/build-source-grammars --wheels-dir /tmp/grammar-wheels
      - name: Verify wheels exist
        run: ls -la /tmp/grammar-wheels/*.whl || echo "No wheels (cache hit expected)"

  pytest:
    needs: [changes, stop-the-line, build-grammars]
    if: always() && needs.changes.outputs.code == 'true' && needs.stop-the-line.result != 'failure' && needs.build-grammars.result != 'failure'
    runs-on: self-hosted
    timeout-minutes: 30
    env:
      SEGMENT_DOWNLOAD_TIMEOUT_MINS: "1"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          # Don't use pip cache - we cache the entire venv instead
      - name: Detect Python prefix
        id: pyprefix
        run: echo "prefix=$(python -c 'import sys; print(sys.prefix)')" >> "$GITHUB_OUTPUT"
      - name: Cache venv
        id: cache-venv
        uses: actions/cache@v3
        with:
          path: .venv
          # Key includes Python prefix to prevent cross-runner cache sharing.
          # Codeberg hosted runners and self-hosted runners install Python at
          # different paths; venv shebangs are not portable across them.
          key: venv-${{ runner.os }}-py3.11-${{ hashFiles('**/pyproject.toml') }}-${{ steps.pyprefix.outputs.prefix }}
      - name: Validate cached venv
        if: steps.cache-venv.outputs.cache-hit == 'true'
        id: validate-venv
        run: |
          # Cached venv may have shebangs from a different runner's Python path.
          # If the venv python is broken, delete it so the install step recreates it.
          if ! .venv/bin/python --version >/dev/null 2>&1; then
            echo "⚠️ Cached venv is broken (wrong Python path) - recreating"
            rm -rf .venv
            echo "venv_ok=false" >> "$GITHUB_OUTPUT"
          else
            echo "✓ Cached venv is functional"
            echo "venv_ok=true" >> "$GITHUB_OUTPUT"
          fi
      - name: Create venv and install dependencies
        if: steps.cache-venv.outputs.cache-hit != 'true' || steps.validate-venv.outputs.venv_ok == 'false'
        run: |
          python -m venv .venv
          source .venv/bin/activate
          pip install --upgrade pip
          # Install from packages if available, otherwise legacy
          if [ -d "packages/hypergumbo-core" ]; then
            # Install test deps directly - NEVER use meta-package (it pulls PyTorch/CUDA)
            pip install pytest pytest-cov pytest-xdist ruff bandit pip-audit jsonschema yamllint
            pip install -e packages/hypergumbo-core -e packages/hypergumbo-lang-mainstream -e packages/hypergumbo-lang-common -e packages/hypergumbo-lang-extended1 -e packages/hypergumbo-lang-rust-analyzer -e "packages/hypergumbo-tracker[dev,tui]"
          else
            pip install -e .[dev]
          fi
      - name: Add venv to PATH
        run: echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH
      - name: Fix pytest wrapper
        run: |
          # pip install pytest overwrites .venv/bin/pytest with its entry point.
          # This repairs it to use smart-test, avoiding re-exec overhead in conftest.py
          ./scripts/install-hooks --repair-shims
      - name: Restore grammar wheels
        uses: actions/cache@v3
        with:
          path: /tmp/grammar-wheels
          key: grammar-wheels-${{ hashFiles('scripts/build-source-grammars') }}
      - name: Install grammars from wheels
        run: ./scripts/build-source-grammars --wheels-dir /tmp/grammar-wheels
      - name: Check for sentence-transformers
        run: |
          # Don't try to install - just check if it's already available.
          # See CONTRIBUTING.md for details on embedding test requirements.
          if python -c "import sentence_transformers" 2>/dev/null; then
            echo "sentence-transformers available"
            echo "EMBEDDINGS_AVAILABLE=true" >> "$GITHUB_ENV"
          else
            echo "sentence-transformers not installed, embedding tests will be skipped"
            echo "EMBEDDINGS_AVAILABLE=false" >> "$GITHUB_ENV"
          fi
      - name: Validate test manifest (smart test selection)
        id: manifest
        run: |
          # Smart test selection: contributor commits .ci/affected-tests.txt
          # CI validates it with cheap sanity checks (no hypergumbo installation)
          # See ADR-0010 for design rationale
          MANIFEST=".ci/affected-tests.txt"

          # For PRs, validate the committed manifest
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE="${{ github.event.pull_request.base.sha }}"
            HEAD="${{ github.event.pull_request.head.sha }}"

            # Release PRs (dev → main) contain cumulative changes already
            # validated per-commit on dev.  The targeted-manifest model
            # cannot cover a full-release diff, so skip pytest here;
            # full-suite.yml validates after merge.
            PR_HEAD_REF="${{ github.head_ref }}"
            if [[ "$PR_HEAD_REF" == release/* ]]; then
              echo "✅ Release branch ($PR_HEAD_REF) — skipping pytest (validated per-commit on dev)"
              echo "skip_tests=true" >> "$GITHUB_OUTPUT"
              exit 0
            fi

            echo "=== Changed files (BASE=$BASE HEAD=$HEAD) ==="
            git diff --name-only "$BASE" "$HEAD" | head -20 || true

            # Check if core infrastructure changed (requires full suite regardless)
            CORE_CHANGED=false
            for core_file in sketch.py slice.py cli.py ir.py all_analyzers.py base.py registry.py; do
              if git diff --name-only "$BASE" "$HEAD" | grep -qE "$core_file$"; then
                CORE_CHANGED=true
                echo "✓ Core file changed: $core_file - will run full suite"
                break
              fi
            done

            # Check if manifest exists (contributor ran scripts/smart-test --manifest locally)
            if [ ! -f "$MANIFEST" ]; then
              echo "❌ No manifest committed"
              echo ""
              echo "Smart test selection requires a manifest. Run locally:"
              echo "  ./scripts/smart-test --manifest"
              echo "  git add .ci/affected-tests.txt"
              echo "  git commit --amend"
              echo ""
              echo "See ADR-0010 for details."
              exit 1
            fi

            echo "=== Validating committed manifest ==="
            # Show header only (not the test list, which is long and confusing when truncated)
            grep "^#" "$MANIFEST" | head -10 || true
            TEST_COUNT=$(grep -c "^packages/" "$MANIFEST" || echo 0)
            echo "# (manifest lists $TEST_COUNT test files)"

            # If core infrastructure changed, manifest should indicate full suite
            if [ "$CORE_CHANGED" = "true" ]; then
              if grep -q "^# Mode: full-suite" "$MANIFEST"; then
                echo "✓ Core changed, manifest indicates full suite - good"
              else
                echo "⚠️ Core infrastructure changed but manifest doesn't indicate full suite"
                echo "   smart-test should have generated a full-suite manifest"
                echo "   Regenerate with: ./scripts/smart-test --manifest"
              fi
            fi

            # Sanity check: verify manifest has at least some tests for changed modules
            # This prevents gaming (e.g., committing empty manifest to skip tests)
            # We use a low minimum (1 test file) because:
            # - Deleting files legitimately reduces test count
            # - smart-test is trusted to generate correct manifests
            # - Coverage check will catch missing coverage anyway
            # NOTE: Only check modules with changed SOURCE files (*.py in src/), not all changed files
            # Config files (pyproject.toml, etc.) don't require tests
            CHANGED_MODULES=$(git diff --name-only "$BASE" "$HEAD" | grep -E "^packages/.*/src/.*\.py$" | cut -d/ -f2 | sort -u || true)

            SANITY_OK=true
            for mod in $CHANGED_MODULES; do
              case $mod in
                hypergumbo)
                  # Meta-package has no tests of its own
                  MIN_TESTS=0 ;;
                *)
                  # At least 1 test file for any changed module
                  MIN_TESTS=1 ;;
              esac

              # Count tests that mention this module
              # Note: grep -c exits 1 when no matches (even though it outputs "0")
              # Using || true prevents the fallback from appending another "0"
              ACTUAL=$(grep -c "packages/$mod/tests/" "$MANIFEST" 2>/dev/null || true)
              ACTUAL=${ACTUAL:-0}

              if [ "$ACTUAL" -lt "$MIN_TESTS" ]; then
                echo "⚠️ Manifest has $ACTUAL tests for $mod, expected ≥$MIN_TESTS"
                SANITY_OK=false
              else
                echo "✓ $mod: $ACTUAL tests"
              fi
            done

            if [ "$SANITY_OK" = "false" ]; then
              echo "⚠️ Manifest sanity check failed"
              echo "   This usually means smart-test wasn't run after changes."
              echo "   Run: pytest  (regenerates manifest)"
              echo "   Then: git add .ci/affected-tests.txt && git commit --amend"
              echo "use_manifest=false" >> "$GITHUB_OUTPUT"
            else
              TOTAL_TESTS=$(grep -c "^packages/" "$MANIFEST" 2>/dev/null || echo 0)
              echo "✅ Manifest validated ($TOTAL_TESTS test files)"
              echo "use_manifest=true" >> "$GITHUB_OUTPUT"
            fi
          else
            # Push events skip pytest in ci.yml - full-suite.yml handles that
            echo "Push event - skipping pytest (full-suite.yml handles comprehensive validation)"
            echo "skip_tests=true" >> "$GITHUB_OUTPUT"
          fi
      - name: Run tests with coverage
        if: steps.manifest.outputs.skip_tests != 'true'
        run: |
          MANIFEST=".ci/affected-tests.txt"
          # Detect available CPUs from cgroup limits (Codeberg sets cpu.max
          # to 200000/100000 = 2 cores but nproc reports the host's 32).
          if [ -f /sys/fs/cgroup/cpu.max ]; then
            read quota period < /sys/fs/cgroup/cpu.max
            if [ "$quota" != "max" ]; then
              WORKERS=$((quota / period))
            else
              WORKERS=$(nproc)
            fi
          else
            WORKERS=$(nproc)
          fi

          if [ ! -f "$MANIFEST" ] || [ "${{ steps.manifest.outputs.use_manifest }}" != "true" ]; then
            echo "❌ No valid manifest - cannot run tests"
            echo ""
            echo "Fast-CI requires a manifest with targeted tests."
            echo "Full suite runs in full-suite.yml after merge, not here."
            exit 1
          fi

          # Read mode from manifest (targeted or full-suite)
          MANIFEST_MODE=$(grep '^# Mode:' "$MANIFEST" | cut -d: -f2 | tr -d ' ' || echo "unknown")
          echo "Manifest mode: $MANIFEST_MODE"

          # Reject full-suite manifests - CI has 5-minute limit, can't run all tests
          # Full suite validation happens in full-suite.yml after merge
          if [ "$MANIFEST_MODE" = "full-suite" ]; then
            REASON=$(grep '^# Reason:' "$MANIFEST" | cut -d: -f2- | sed 's/^ *//' || echo "unknown")
            echo ""
            echo "❌ Fast-CI cannot run full-suite manifests"
            echo "   Manifest mode: full-suite"
            echo "   Reason: $REASON"
            echo ""
            echo "CI has a 5-minute time limit. Full test suite runs after merge via full-suite.yml."
            echo ""
            echo "To fix, regenerate the manifest with targeted tests:"
            echo "  pytest              # runs affected tests, updates targeted manifest"
            echo "  git add .ci/affected-tests.txt"
            echo "  git commit --amend"
            echo ""
            echo "Note: 'pytest --full' runs all tests locally but still writes a targeted manifest."
            exit 1
          fi

          # Read changed source files from manifest (between markers)
          CHANGED_SOURCE_FILES=$(sed -n '/^# === CHANGED_SOURCE_FILES ===/,/^# === SELECTED_TESTS ===/p' "$MANIFEST" | grep -v '^#' | grep -v '^$' || echo "")

          # Sanity check: verify manifest's changed source files match actual PR diff
          BASE="${{ github.event.pull_request.base.sha }}"
          HEAD="${{ github.event.pull_request.head.sha }}"
          ACTUAL_SOURCE_FILES=$(git diff --name-only "$BASE" "$HEAD" | grep -E '^packages/.*/src/.*\.py$' | sort || echo "")
          MANIFEST_SOURCE_FILES=$(echo "$CHANGED_SOURCE_FILES" | sort || echo "")

          echo "=== Sanity check: changed source files ==="
          echo "From manifest: $(echo "$MANIFEST_SOURCE_FILES" | wc -w) files"
          echo "From PR diff:  $(echo "$ACTUAL_SOURCE_FILES" | wc -w) files"

          # If no Python source files changed, skip tests entirely
          # This handles PRs that only change shell scripts, YAML, docs, etc.
          # Full suite runs in full-suite.yml after merge to catch any issues
          if [ -z "$ACTUAL_SOURCE_FILES" ] && [ -z "$MANIFEST_SOURCE_FILES" ]; then
            echo ""
            echo "✅ No Python source files changed - skipping pytest"
            echo "   Changes are to infrastructure/config files only."
            echo "   Full suite will run via full-suite.yml after merge."
            exit 0
          fi

          # Check that manifest includes all files from PR diff
          # (manifest may include more if baseline differs, but should not miss any)
          MISSING=""
          for f in $ACTUAL_SOURCE_FILES; do
            if ! echo "$MANIFEST_SOURCE_FILES" | grep -qF "$f"; then
              MISSING="$MISSING $f"
            fi
          done

          if [ -n "$MISSING" ]; then
            echo "⚠️ Manifest missing source files from PR diff:$MISSING"
            echo ""
            echo "This typically happens when pytest was run before staging changes."
            echo "The manifest was generated without seeing the staged files."
            echo ""
            echo "To fix: stage changes first, then run pytest"
            echo "  git add <files>"
            echo "  pytest  # regenerates manifest with correct file list"
            echo "  git commit"
            echo ""
            echo "Continuing with PR diff files for coverage check..."
            CHANGED_SOURCE_FILES="$ACTUAL_SOURCE_FILES"
          else
            echo "✓ Manifest includes all changed source files"
          fi

          # Build coverage paths for affected packages only
          if [ -n "$CHANGED_SOURCE_FILES" ]; then
            PKG_DIRS=$(echo "$CHANGED_SOURCE_FILES" | sed 's|^\(packages/[^/]*/src\)/.*|\1|' | sort -u)
            COV_PATHS=""
            for pkg in $PKG_DIRS; do
              COV_PATHS="$COV_PATHS --cov=$pkg"
            done
            # Suppress terminal report - we'll show targeted coverage below
            COV_PATHS="$COV_PATHS --cov-report="
            # Use no-embeddings config since sentence-transformers isn't installed in CI
            COV_PATHS="$COV_PATHS --cov-config=.coveragerc.no-embeddings"
          else
            # No source files changed - skip coverage
            COV_PATHS=""
          fi

          # Filter out files that are intentionally excluded from the 100%-coverage
          # contract:
          #   - embedding modules (require sentence-transformers, absent in CI)
          #   - anything under a /_generated/ directory (protoc / protobuf output,
          #     regenerated by scripts/build-scip-proto and similar helpers)
          # These are also covered by .coveragerc / .coveragerc.no-embeddings omit
          # lists, but the --include filter below bypasses omit, so we must skip
          # them explicitly here too.
          EMBEDDING_FILES="sketch_embeddings.py _embedding_data.py"
          COVERABLE_SOURCE_FILES=""
          for f in $CHANGED_SOURCE_FILES; do
            IS_EMBEDDING=false
            for ef in $EMBEDDING_FILES; do
              if echo "$f" | grep -qE "/$ef$"; then
                IS_EMBEDDING=true
                echo "  (skipping embedding file from coverage check: $f)"
                break
              fi
            done
            if echo "$f" | grep -qE "/_generated/"; then
              IS_EMBEDDING=true
              echo "  (skipping auto-generated file from coverage check: $f)"
            fi
            if [ "$IS_EMBEDDING" = "false" ]; then
              COVERABLE_SOURCE_FILES="$COVERABLE_SOURCE_FILES $f"
            fi
          done
          COVERABLE_SOURCE_FILES=$(echo "$COVERABLE_SOURCE_FILES" | xargs)  # trim whitespace

          echo ""
          echo "=== Python version ==="
          python --version

          echo "=== Installed packages ==="
          pip list | grep -E "hypergumbo|pytest|tree-sitter" | head -20 || true

          echo ""
          # Read selected tests from manifest (after SELECTED_TESTS marker)
          SELECTED_TESTS=$(sed -n '/^# === SELECTED_TESTS ===/,$p' "$MANIFEST" | grep -v '^#' | grep -v '^$' | tr '\n' ' ')
          TEST_COUNT=$(echo "$SELECTED_TESTS" | wc -w)

          if [ "$MANIFEST_MODE" = "full-suite" ]; then
            REASON=$(grep '^# Reason:' "$MANIFEST" | cut -d: -f2- | sed 's/^ *//' || echo "unknown")
            echo "=== Running full test suite ($REASON) ==="
          else
            echo "=== Running $TEST_COUNT selected tests (smart test selection) ==="
          fi

          echo "Running: pytest ($TEST_COUNT test files)"
          # SMART_TEST_ACTIVE=1 bypasses smart-test wrapper to use real pytest
          # (CI already computed test selection via manifest - don't re-compute)
          # --rootdir=. pins pytest to repo root so manifest paths resolve
          # correctly even for single-package PRs (see ADR-0010 §Rootdir Pinning)
          SMART_TEST_ACTIVE=1 pytest --rootdir=. -n "$WORKERS" --tb=short $COV_PATHS $SELECTED_TESTS
          TEST_EXIT=$?

          # Check 100% coverage for changed source files only
          # (COVERABLE_SOURCE_FILES excludes embedding files that require sentence-transformers)
          if [ $TEST_EXIT -eq 0 ] && [ -n "$COVERABLE_SOURCE_FILES" ]; then
            echo ""
            echo "=== Checking 100% coverage for changed source files ==="

            # Filter out deleted files (they won't exist and can't have coverage)
            EXISTING_FILES=""
            for f in $COVERABLE_SOURCE_FILES; do
              if [ -f "$f" ]; then
                EXISTING_FILES="$EXISTING_FILES $f"
              else
                echo "  (skipping deleted file: $f)"
              fi
            done
            EXISTING_FILES=$(echo "$EXISTING_FILES" | xargs)  # trim whitespace

            if [ -z "$EXISTING_FILES" ]; then
              echo "✅ All changed files were deleted - no coverage check needed"
            else
              INCLUDE_PATTERN=$(echo "$EXISTING_FILES" | tr ' ' ',' | sed 's/,$//')

              # Use no-embeddings rcfile so exclude_also patterns are respected
              COV_RCFILE="--rcfile=.coveragerc.no-embeddings"
              if ! coverage report $COV_RCFILE --include="$INCLUDE_PATTERN" --fail-under=100; then
                echo ""
                echo "❌ Changed source files are not at 100% coverage:"
                coverage report $COV_RCFILE --include="$INCLUDE_PATTERN" --show-missing
                exit 1
              else
                echo "✅ All changed source files are at 100% coverage"
              fi
            fi
          fi

          exit $TEST_EXIT

      # Coverage badge is updated by full-suite.yml, not here
      # (ci.yml skips pytest on push events)

  # Retry job for pytest when Codeberg's hard 5-minute cap kills the primary.
  # Runs on self-hosted runner (no time limit). Analogous to full-suite.yml retries.
  # always() ensures the if-condition is evaluated even when pytest is cancelled
  # (without it, Forgejo skips the dependent entirely).
  pytest-retry:
    needs: [changes, stop-the-line, build-grammars, pytest]
    if: >-
      always() &&
      needs.changes.outputs.code == 'true' &&
      needs.stop-the-line.result != 'failure' &&
      needs.build-grammars.result != 'failure' &&
      (needs.pytest.result == 'failure' || needs.pytest.result == 'cancelled')
    runs-on: self-hosted
    timeout-minutes: 15
    env:
      SEGMENT_DOWNLOAD_TIMEOUT_MINS: "1"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Detect Python prefix
        id: pyprefix
        run: echo "prefix=$(python -c 'import sys; print(sys.prefix)')" >> "$GITHUB_OUTPUT"
      - name: Cache venv
        id: cache-venv
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-py3.11-${{ hashFiles('**/pyproject.toml') }}-${{ steps.pyprefix.outputs.prefix }}
      - name: Validate cached venv
        if: steps.cache-venv.outputs.cache-hit == 'true'
        id: validate-venv
        run: |
          if ! .venv/bin/python --version >/dev/null 2>&1; then
            echo "⚠️ Cached venv is broken (wrong Python path) - recreating"
            rm -rf .venv
            echo "venv_ok=false" >> "$GITHUB_OUTPUT"
          else
            echo "✓ Cached venv is functional"
            echo "venv_ok=true" >> "$GITHUB_OUTPUT"
          fi
      - name: Create venv and install dependencies
        if: steps.cache-venv.outputs.cache-hit != 'true' || steps.validate-venv.outputs.venv_ok == 'false'
        run: |
          python -m venv .venv
          source .venv/bin/activate
          pip install --upgrade pip
          if [ -d "packages/hypergumbo-core" ]; then
            pip install pytest pytest-cov pytest-xdist ruff bandit pip-audit jsonschema yamllint
            pip install -e packages/hypergumbo-core -e packages/hypergumbo-lang-mainstream -e packages/hypergumbo-lang-common -e packages/hypergumbo-lang-extended1 -e packages/hypergumbo-lang-rust-analyzer -e "packages/hypergumbo-tracker[dev,tui]"
          else
            pip install -e .[dev]
          fi
      - name: Add venv to PATH
        run: echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH
      - name: Fix pytest wrapper
        run: ./scripts/install-hooks --repair-shims
      - name: Restore grammar wheels
        uses: actions/cache@v3
        with:
          path: /tmp/grammar-wheels
          key: grammar-wheels-${{ hashFiles('scripts/build-source-grammars') }}
      - name: Install grammars from wheels
        run: ./scripts/build-source-grammars --wheels-dir /tmp/grammar-wheels
      - name: Check for sentence-transformers
        run: |
          if python -c "import sentence_transformers" 2>/dev/null; then
            echo "sentence-transformers available"
            echo "EMBEDDINGS_AVAILABLE=true" >> "$GITHUB_ENV"
          else
            echo "sentence-transformers not installed, embedding tests will be skipped"
            echo "EMBEDDINGS_AVAILABLE=false" >> "$GITHUB_ENV"
          fi
      - name: Validate test manifest (smart test selection)
        id: manifest
        run: |
          MANIFEST=".ci/affected-tests.txt"

          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE="${{ github.event.pull_request.base.sha }}"
            HEAD="${{ github.event.pull_request.head.sha }}"

            # Release PRs skip pytest (same logic as primary job)
            PR_HEAD_REF="${{ github.head_ref }}"
            if [[ "$PR_HEAD_REF" == release/* ]]; then
              echo "✅ Release branch ($PR_HEAD_REF) — skipping pytest"
              echo "skip_tests=true" >> "$GITHUB_OUTPUT"
              exit 0
            fi

            if [ ! -f "$MANIFEST" ]; then
              echo "❌ No manifest committed"
              exit 1
            fi

            echo "=== Validating committed manifest (retry) ==="
            grep "^#" "$MANIFEST" | head -10 || true
            TEST_COUNT=$(grep -c "^packages/" "$MANIFEST" || echo 0)
            echo "# (manifest lists $TEST_COUNT test files)"

            if grep -q "^# Mode: full-suite" "$MANIFEST"; then
              echo "❌ Fast-CI cannot run full-suite manifests"
              exit 1
            fi

            echo "✅ Manifest validated"
            echo "use_manifest=true" >> "$GITHUB_OUTPUT"
          else
            echo "Push event - skipping pytest"
            echo "skip_tests=true" >> "$GITHUB_OUTPUT"
          fi
      - name: Run tests with coverage
        if: steps.manifest.outputs.skip_tests != 'true'
        run: |
          MANIFEST=".ci/affected-tests.txt"
          if [ -f /sys/fs/cgroup/cpu.max ]; then
            read quota period < /sys/fs/cgroup/cpu.max
            if [ "$quota" != "max" ]; then
              WORKERS=$((quota / period))
            else
              WORKERS=$(nproc)
            fi
          else
            WORKERS=$(nproc)
          fi

          if [ ! -f "$MANIFEST" ] || [ "${{ steps.manifest.outputs.use_manifest }}" != "true" ]; then
            echo "❌ No valid manifest - cannot run tests"
            exit 1
          fi

          MANIFEST_MODE=$(grep '^# Mode:' "$MANIFEST" | cut -d: -f2 | tr -d ' ' || echo "unknown")

          BASE="${{ github.event.pull_request.base.sha }}"
          HEAD="${{ github.event.pull_request.head.sha }}"

          CHANGED_SOURCE_FILES=$(sed -n '/^# === CHANGED_SOURCE_FILES ===/,/^# === SELECTED_TESTS ===/p' "$MANIFEST" | grep -v '^#' | grep -v '^$' || echo "")
          ACTUAL_SOURCE_FILES=$(git diff --name-only "$BASE" "$HEAD" | grep -E '^packages/.*/src/.*\.py$' | sort || echo "")

          if [ -z "$ACTUAL_SOURCE_FILES" ] && [ -z "$CHANGED_SOURCE_FILES" ]; then
            echo "✅ No Python source files changed - skipping pytest"
            exit 0
          fi

          # Use actual PR diff files for coverage if manifest is missing any
          MISSING=""
          MANIFEST_SOURCE_FILES=$(echo "$CHANGED_SOURCE_FILES" | sort || echo "")
          for f in $ACTUAL_SOURCE_FILES; do
            if ! echo "$MANIFEST_SOURCE_FILES" | grep -qF "$f"; then
              MISSING="$MISSING $f"
            fi
          done
          if [ -n "$MISSING" ]; then
            CHANGED_SOURCE_FILES="$ACTUAL_SOURCE_FILES"
          fi

          if [ -n "$CHANGED_SOURCE_FILES" ]; then
            PKG_DIRS=$(echo "$CHANGED_SOURCE_FILES" | sed 's|^\(packages/[^/]*/src\)/.*|\1|' | sort -u)
            COV_PATHS=""
            for pkg in $PKG_DIRS; do
              COV_PATHS="$COV_PATHS --cov=$pkg"
            done
            COV_PATHS="$COV_PATHS --cov-report= --cov-config=.coveragerc.no-embeddings"
          else
            COV_PATHS=""
          fi

          # Retry path — same skip policy as pytest job above.
          EMBEDDING_FILES="sketch_embeddings.py _embedding_data.py"
          COVERABLE_SOURCE_FILES=""
          for f in $CHANGED_SOURCE_FILES; do
            IS_EMBEDDING=false
            for ef in $EMBEDDING_FILES; do
              if echo "$f" | grep -qE "/$ef$"; then
                IS_EMBEDDING=true
                break
              fi
            done
            if echo "$f" | grep -qE "/_generated/"; then
              IS_EMBEDDING=true
            fi
            if [ "$IS_EMBEDDING" = "false" ]; then
              COVERABLE_SOURCE_FILES="$COVERABLE_SOURCE_FILES $f"
            fi
          done
          COVERABLE_SOURCE_FILES=$(echo "$COVERABLE_SOURCE_FILES" | xargs)

          SELECTED_TESTS=$(sed -n '/^# === SELECTED_TESTS ===/,$p' "$MANIFEST" | grep -v '^#' | grep -v '^$' | tr '\n' ' ')
          TEST_COUNT=$(echo "$SELECTED_TESTS" | wc -w)

          echo "=== Running $TEST_COUNT selected tests (retry on self-hosted) ==="
          SMART_TEST_ACTIVE=1 pytest --rootdir=. -n "$WORKERS" --tb=short $COV_PATHS $SELECTED_TESTS
          TEST_EXIT=$?

          if [ $TEST_EXIT -eq 0 ] && [ -n "$COVERABLE_SOURCE_FILES" ]; then
            echo ""
            echo "=== Checking 100% coverage for changed source files ==="
            EXISTING_FILES=""
            for f in $COVERABLE_SOURCE_FILES; do
              if [ -f "$f" ]; then
                EXISTING_FILES="$EXISTING_FILES $f"
              fi
            done
            EXISTING_FILES=$(echo "$EXISTING_FILES" | xargs)

            if [ -z "$EXISTING_FILES" ]; then
              echo "✅ All changed files were deleted - no coverage check needed"
            else
              INCLUDE_PATTERN=$(echo "$EXISTING_FILES" | tr ' ' ',' | sed 's/,$//')
              # Use no-embeddings rcfile so exclude_also patterns are respected
              COV_RCFILE="--rcfile=.coveragerc.no-embeddings"
              if ! coverage report $COV_RCFILE --include="$INCLUDE_PATTERN" --fail-under=100; then
                echo ""
                echo "❌ Changed source files are not at 100% coverage:"
                coverage report $COV_RCFILE --include="$INCLUDE_PATTERN" --show-missing
                exit 1
              else
                echo "✅ All changed source files are at 100% coverage"
              fi
            fi
          fi

          exit $TEST_EXIT
      - name: Clear failed pytest status
        if: success()
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # When pytest-retry succeeds, overwrite the failed "pytest" commit
          # status so Forgejo branch protection sees all checks as passed.
          # Without this, the original pytest failure blocks merge even though
          # ci-complete reports success.
          SHA="${{ github.event.pull_request.head.sha || github.sha }}"
          API_BASE="${{ github.server_url }}/api/v1/repos/${{ github.repository }}"
          curl -sf -X POST \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Content-Type: application/json" \
            -d '{
              "context": "CI / pytest (pull_request)",
              "description": "Tests passed on retry (self-hosted)",
              "state": "success",
              "target_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
            }' \
            "${API_BASE}/statuses/${SHA}" || echo "Warning: failed to update commit status (non-fatal)"

  tracker-validate:
    needs: [changes, stop-the-line]
    if: >-
      always() &&
      needs.changes.outputs.tracker_data == 'true' &&
      needs.stop-the-line.result != 'failure'
    runs-on: self-hosted
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install tracker
        run: |
          pip install --upgrade pip
          pip install -e packages/hypergumbo-tracker
      - name: Validate tracker data
        run: scripts/tracker validate

  # Final gate job - ensures workflow completes even when jobs are skipped
  # This fixes Forgejo Actions bug where skipped jobs leave workflow in "running" state
  ci-complete:
    needs: [changes, stop-the-line, lint, hook-tests, audit, verify-generated, build-grammars, pytest, pytest-retry, dco, tracker-validate]
    if: always()
    runs-on: self-hosted
    timeout-minutes: 5
    steps:
      - name: Check job results
        run: |
          echo "Job results:"
          echo "  changes: ${{ needs.changes.result }}"
          echo "  stop-the-line: ${{ needs.stop-the-line.result }}"
          echo "  lint: ${{ needs.lint.result }}"
          echo "  hook-tests: ${{ needs.hook-tests.result }}"
          echo "  audit: ${{ needs.audit.result }}"
          echo "  verify-generated: ${{ needs.verify-generated.result }}"
          echo "  build-grammars: ${{ needs.build-grammars.result }}"
          echo "  pytest: ${{ needs.pytest.result }}"
          echo "  pytest-retry: ${{ needs.pytest-retry.result }}"
          echo "  dco: ${{ needs.dco.result }}"
          echo "  tracker-validate: ${{ needs.tracker-validate.result }}"

          # pytest passes if primary OR retry succeeded
          pytest_ok=false
          [[ "${{ needs.pytest.result }}" == "success" ]] && pytest_ok=true
          [[ "${{ needs.pytest-retry.result }}" == "success" ]] && pytest_ok=true
          # If pytest was skipped (no code changes), that's OK too
          [[ "${{ needs.pytest.result }}" == "skipped" ]] && pytest_ok=true

          # Fail if any required job failed (skipped is OK)
          if [[ "${{ needs.changes.result }}" == "failure" ]] || \
             [[ "${{ needs.stop-the-line.result }}" == "failure" ]] || \
             [[ "${{ needs.lint.result }}" == "failure" ]] || \
             [[ "${{ needs.hook-tests.result }}" == "failure" ]] || \
             [[ "${{ needs.audit.result }}" == "failure" ]] || \
             [[ "${{ needs.verify-generated.result }}" == "failure" ]] || \
             [[ "$pytest_ok" != "true" ]] || \
             [[ "${{ needs.dco.result }}" == "failure" ]] || \
             [[ "${{ needs.tracker-validate.result }}" == "failure" ]]; then
            echo "❌ One or more jobs failed"
            exit 1
          fi

          echo "✅ All required jobs passed or were skipped"