ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    # Nightly 04:17 UTC — runs the full suite on main to catch dependency
    # drift and flakiness without waiting for a PR. Sits just before the
    # no-org-leak cron at 04:27.
    - cron: "17 4 * * *"
  workflow_dispatch:

# Cancel superseded runs for the same ref; keep main runs to preserve signals.
concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # ── validate: lockfile, config, generated files, docs, metadata ────────
  validate:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          enable-cache: true

      - name: Verify lockfile is up to date
        run: uv lock --check

      - name: Validate TOML/YAML/JSON
        run: |
          uv run python - <<'PY'
          import json, tomllib, sys
          from pathlib import Path
          def toml(p):
              with open(p, "rb") as f:
                  tomllib.load(f)
          toml("pyproject.toml")
          toml("uv.lock")
          for p in Path(".").rglob("*.json"):
              if ".venv" in p.parts or "node_modules" in p.parts:
                  continue
              try:
                  json.loads(p.read_text())
              except json.JSONDecodeError as e:
                  print(f"invalid json: {p}: {e}", file=sys.stderr); sys.exit(1)
          import glob
          for p in glob.glob(".github/**/*.yml", recursive=True):
              import yaml  # noqa: F401 (best-effort; yaml may be absent)
          print("config validation ok")
          PY

      - name: OpenAPI snapshot check
        run: uv run python -c "import kater.openapi_spec as s; assert s.generate_spec(); print('openapi ok')"

      - name: Lint workflow files
        run: |
          for f in .github/workflows/*.yml; do
            python - "$f" <<'PY'
          import sys, yaml
          with open(sys.argv[1]) as f:
              yaml.safe_load(f)
          print("workflow ok:", sys.argv[1])
          PY
          done

      - name: Docs and package metadata check
        run: |
          uv run python - <<'PY'
          import tomllib
          with open("pyproject.toml", "rb") as f:
              meta = tomllib.load(f)["project"]
          assert meta.get("version"), "missing version"
          assert meta.get("readme"), "missing readme"
          print("metadata ok:", meta["name"], meta["version"])
          PY

      - name: Agent-taste eval gate
        run: |
          uv sync --frozen --dev
          uv run python .agents/scripts/generate-taste.py --check
          uv run python .agents/scripts/eval-score.py --gate

      - name: Cursor artifact guard
        run: bash scripts/check_cursor_artifacts.sh

  # ── lint-type: ruff, format, mypy on the 3.11 syntax baseline ──────────
  lint-type:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: "3.11"
          enable-cache: true

      - name: Sync dependencies (frozen)
        run: uv sync --frozen --dev --extra browser

      - name: Ruff
        run: uv run ruff check .

      - name: Mypy
        run: uv run mypy

  # ── unit: matrix across supported Python versions ─────────────────────
  unit:
    needs: [validate]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13", "3.14"]
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: ${{ matrix.python-version }}
          enable-cache: true

      - name: Sync dependencies (frozen)
        run: uv sync --frozen --dev --extra browser

      # The inner `timeout` is a fast, explicit guard that yields a clean
      # exit 124 if a test wedges; the job-level `timeout-minutes: 15` above is
      # the real backstop against hangs. Keep this comfortably above the actual
      # suite runtime: 3.11/3.12 finish ~150s and 3.13 ~157s, but 3.14 runs
      # ~20-30s slower and was tripping a 180s cap even though the full suite
      # (see the coverage job, with instrumentation) completes in ~185s.
      - name: Test
        run: >-
          timeout 300s uv run pytest -vv --tb=short -ra
          --ignore=tests/test_computer_acceptance_e2e.py

  # ── integration: REST, OAuth PKCE, API-key, admin-key, storage, proxy ──
  integration:
    needs: [validate]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: "3.12"
          enable-cache: true

      - name: Sync dependencies (frozen)
        run: uv sync --frozen --dev --extra browser

      - name: Integration tests
        run: timeout 180s uv run pytest -vv --tb=short -ra tests/test_api.py tests/test_oauth.py tests/test_authgate.py tests/test_storage.py tests/test_proxy.py

  # The Computer lane is a cross-repository contract gate. Keep it out of the
  # Python-version matrix and run it once against the exact private contract/runtime
  # commit that Kater's generated package was synchronized from. Dependabot
  # cannot receive the private deploy key, so only that actor gets an explicit,
  # auditable skip while the public unit/integration/e2e gates remain required.
  computer-acceptance:
    needs: [validate]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Explain skipped private acceptance
        if: github.event_name == 'pull_request' && github.actor == 'dependabot[bot]'
        run: |
          echo "Private computer-acceptance skipped for Dependabot because GitHub withholds repository secrets from Dependabot-triggered workflows." | tee -a "$GITHUB_STEP_SUMMARY"

      - name: Explain skipped fork acceptance
        if: github.event.repository.fork == true
        run: |
          echo "Private UDO acceptance skipped outside the upstream repository; the public Kater gates remain required." | tee -a "$GITHUB_STEP_SUMMARY"

      - name: Checkout Kater
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Checkout pinned private contract runtime
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        with:
          repository: ${{ github.repository_owner }}/utrecht-data-os
          ref: 4fc8217eb90d1a6024030620e35f84369bb5c805
          path: utrecht-data-os
          ssh-key: ${{ secrets.UDO_READ_DEPLOY_KEY }}
          persist-credentials: false

      - name: Install uv
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: "3.12"
          enable-cache: true

      - name: Sync Kater dependencies (frozen)
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        run: uv sync --frozen --dev

      - name: Install pinned controller dependencies
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        working-directory: utrecht-data-os/platform/agent-control-plane/apps/session-controller
        run: npm ci --ignore-scripts

      - name: Run Computer acceptance lane
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        env:
          COMPUTER_ACCEPTANCE_EVIDENCE: ${{ runner.temp }}/computer-acceptance-evidence.json
          KATER_ACCEPTANCE_CHECKOUT: ${{ github.workspace }}/utrecht-data-os
        run: timeout 180s uv run pytest -vv --tb=short -ra tests/test_computer_acceptance_e2e.py

      - name: Publish schema-validated acceptance evidence
        if: github.event.repository.fork != true && (github.event_name != 'pull_request' || github.actor != 'dependabot[bot]')
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: computer-acceptance-evidence
          path: ${{ runner.temp }}/computer-acceptance-evidence.json
          if-no-files-found: error

  # ── e2e: real Kater processes (health, REST, MCP, SSE, WS, dashboard) ──
  e2e:
    needs: [unit, integration, computer-acceptance]
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          python-version: "3.12"
          enable-cache: true

      - name: Sync dependencies (frozen)
        run: uv sync --frozen --dev --extra browser

      - name: Start Kater
        run: |
          uv run kater serve &
          echo $! > /tmp/kater.pid
          for i in $(seq 1 30); do
            if curl -fsS http://127.0.0.1:9091/health >/dev/null 2>&1; then
              echo "kater healthy"; break
            fi
            sleep 1
          done

      - name: End-to-end MCP check
        run: ./scripts/e2e-mcp.sh

      - name: Stop Kater
        if: always()
        run: kill "$(cat /tmp/kater.pid)" 2>/dev/null || true

      - name: Smoke test
        run: ./scripts/smoke.sh

  # ── package: wheel/sdist build + clean-venv install smoke ─────────────
  package:
    needs: [validate]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          enable-cache: true

      - name: Build wheel and sdist
        run: uv build

      - name: Install into a clean venv and smoke
        run: |
          uv venv /tmp/kater-pkg-test
          uv pip install --python /tmp/kater-pkg-test dist/*.whl
          /tmp/kater-pkg-test/bin/kater version
          /tmp/kater-pkg-test/bin/kater doctor
          /tmp/kater-pkg-test/bin/kater --help

      - name: Verify package contents
        run: |
          python - <<'PY'
          import zipfile, glob, sys
          wheels = glob.glob("dist/*.whl")
          assert wheels, "no wheel built"
          names = zipfile.ZipFile(wheels[0]).namelist()
          missing = [n for n in ("kater/web/dashboard.py",) if not any(x.endswith(n) for x in names)]
          assert not missing, f"missing packaged files: {missing}"
          print("package contents ok")
          PY

  # ── security-pr: supply chain + secret scanning ───────────────────────
  # (CodeQL is already enforced repo-wide via default setup scanning
  #  python + actions, so we don't upload a second SARIF here.)
  security-pr:
    needs: [validate]
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          enable-cache: true

      - name: Dependency review
        if: github.event.repository.fork != true && github.event_name == 'pull_request'
        uses: actions/dependency-review-action@cc4f6536e38d1126c5e3b0683d469a14f23bfea4 # v3
        with:
          fail-on-severity: high

      - name: Ruff security rules
        run: uv run ruff check --select S .

      - name: Dependency vulnerability audit
        run: uv run pip-audit

      - name: Diff-based secret scan
        run: |
          uv run python - <<'PY'
          import subprocess, sys
          base = "origin/main"
          files = subprocess.run(
              ["git", "diff", "--name-only", base, "HEAD"],
              capture_output=True, text=True,
          ).stdout.split()
          sensitive = ("test", "fixture", "docs", "example")
          targets = [f for f in files if not any(s in f.lower() for s in sensitive)]
          if not targets:
              print("no scan targets"); sys.exit(0)
          res = subprocess.run(["git", "grep", "-IEn",
              r"(?i)(api[_-]?key|secret|token|password|passwd)\s*[=:]\s*['\"][A-Za-z0-9_\-]{8,}",
              "--"] + targets, capture_output=True, text=True)
          if res.returncode == 0:
              print("POSSIBLE SECRET IN DIFF:\n" + res.stdout)
              sys.exit(1)
          print("diff secret scan ok")
          PY

  # ── coverage: project-wide coverage gate (runs the full test suite) ──
  coverage:
    needs: [validate]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false
      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          enable-cache: true
      - name: Sync project
        run: uv sync --frozen --dev --extra browser
      - name: Install Playwright Chromium
        run: uv run playwright install chromium
      - name: Run full test suite with coverage gate
        run: |
          uv run pytest \
            --ignore=tests/test_computer_acceptance_e2e.py \
            --cov-fail-under=25 \
            --cov-report=xml::coverage.xml \
            --cov-report=term-missing:skip-covered
      - name: Upload coverage XML
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-xml
          path: coverage.xml
          if-no-files-found: warn

  # ── gate: single stable required check aggregating the mandatory jobs ──
  gate:
    needs:
      - validate
      - lint-type
      - unit
      - integration
      - e2e
      - package
      - security-pr
      - coverage
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: always()
    steps:
      - name: Aggregate required checks
        run: |
          echo "validate=${{ needs.validate.result }}"
          echo "lint-type=${{ needs.lint-type.result }}"
          echo "unit=${{ needs.unit.result }}"
          echo "integration=${{ needs.integration.result }}"
          echo "e2e=${{ needs.e2e.result }}"
          echo "package=${{ needs.package.result }}"
          echo "security-pr=${{ needs.security-pr.result }}"
          echo "coverage=${{ needs.coverage.result }}"
          for r in "${{ needs.validate.result }}" "${{ needs.lint-type.result }}" \
                   "${{ needs.unit.result }}" "${{ needs.integration.result }}" \
                   "${{ needs.e2e.result }}" "${{ needs.package.result }}" \
                   "${{ needs.security-pr.result }}" "${{ needs.coverage.result }}"; do
            if [ "$r" != "success" ]; then
              echo "required job failed: $r"; exit 1
            fi
          done
          echo "gate: PASS"

  # ── summary: always-on job with a per-job conclusions table + triage links ──
  # Runs even when gate fails, so the PR/failure notification surfaces a single
  # at-a-glance view instead of forcing reviewers to click through every job.
  summary:
    needs:
      - validate
      - lint-type
      - unit
      - integration
      - computer-acceptance
      - e2e
      - package
      - security-pr
      - gate
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: always()
    permissions:
      contents: read
    steps:
      - name: Publish CI summary
        run: |
          cat <<'EOF' >> "$GITHUB_STEP_SUMMARY"
          ## CI summary

          | job | conclusion |
          | --- | --- |
          | validate | ${{ needs.validate.result }} |
          | lint-type | ${{ needs.lint-type.result }} |
          | unit | ${{ needs.unit.result }} |
          | integration | ${{ needs.integration.result }} |
          | computer-acceptance | ${{ needs.computer-acceptance.result }} |
          | e2e | ${{ needs.e2e.result }} |
          | package | ${{ needs.package.result }} |
          | security-pr | ${{ needs.security-pr.result }} |
          | gate | ${{ needs.gate.result }} |

          ### Run context

          - **commit:** ${{ github.sha }}
          - **branch:** ${{ github.ref_name }}
          - **event:** ${{ github.event_name }}
          - **workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          EOF