ci.yml

name: CI

on:
  push:
    branches:
      - main
      - develop
  pull_request:
    branches:
      - main
      - develop
  workflow_dispatch:

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"

jobs:
  # FAST GATES
  lint:
    name: "Lint (ruff check + format)"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Cache uv
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/uv
          key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
          restore-keys: uv-${{ runner.os }}-

      - name: Install uv
        shell: bash
        run: |
          set -euo pipefail
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Sync dependencies
        run: uv sync --frozen

      - name: Ruff check
        run: uv run ruff check services libs

      - name: Ruff format check
        run: uv run ruff format --check services libs

  action-ref-validation:
    name: "Validate action refs are immutable"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Fail on mutable action refs
        shell: bash
        run: |
          set -euo pipefail
          mapfile -t workflow_files < <(find .github/workflows -maxdepth 1 -type f \( -name '*.yml' -o -name '*.yaml' \) | sort)
          if [ ${#workflow_files[@]} -eq 0 ]; then
            echo "No workflow files found."
            exit 0
          fi
          mutable_refs=$(grep -nE 'uses:\s*[^#]+@(v[0-9]+|latest|main|master|develop)\b' "${workflow_files[@]}" || true)
          if [ -n "$mutable_refs" ]; then
            echo "Mutable GitHub Action references detected. Pin actions to full commit SHAs."
            echo "$mutable_refs"
            exit 1
          fi
          echo "All action references are pinned to immutable SHAs."

  # UNIT & DEPENDENCY SECURITY
  unit-tests:
    name: "Unit tests (sqlite)"
    runs-on: ubuntu-latest
    needs: [lint, action-ref-validation]
    timeout-minutes: 10
    env:
      DATABASE_URL_TEST: "sqlite+aiosqlite:///:memory:"
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Cache uv
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/uv
          key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
          restore-keys: uv-${{ runner.os }}-

      - name: Install uv
        shell: bash
        run: |
          set -euo pipefail
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Sync dependencies
        run: uv sync --frozen

      - name: Run unit tests
        run: uv run pytest -q -m unit

  python-deps:
    name: "Python dependencies (pip-audit)"
    needs: [lint, action-ref-validation]
    uses: ./.github/workflows/pip-audit.yml
    with:
      ignore-vulns: |
        PYSEC-2026-161
        CVE-2026-34993
        CVE-2026-47265
        CVE-2026-54283
        CVE-2026-54282
        PYSEC-2026-179
        PYSEC-2026-175
        PYSEC-2026-177
        PYSEC-2026-178

  codeql:
    name: "CodeQL analyze (Python)"
    runs-on: ubuntu-latest
    needs: [lint, action-ref-validation]
    permissions:
      actions: read
      contents: read
      security-events: write
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Initialize CodeQL
        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
        with:
          languages: python

      - name: Autobuild
        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2

      - name: Analyze
        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
        with:
          upload: always
          category: /language:python

  # DOCKER BUILD, PUSH & SCAN (Azure Container Registry)
  docker-build:
    name: "Build and push candidate image (ingestor)"
    runs-on: ubuntu-latest
    needs: [lint, unit-tests, python-deps, action-ref-validation, codeql, integration-tests]
    if: vars.ACR_LOGIN_SERVER != ''
    timeout-minutes: 20
    permissions:
      contents: read
    outputs:
      image_ingestor: ${{ steps.build-ingestor.outputs.image }}
      tree_sha: ${{ steps.resolve.outputs.tree_sha }}
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Resolve tree SHA
        id: resolve
        uses: ./.github/actions/resolve-tree-sha

      - name: Log in to ACR
        uses: docker/login-action@17f7c1809f7467d4244b4ff01b0dbe0b8b9e3d43 # v3.3.0
        with:
          registry: ${{ vars.ACR_LOGIN_SERVER }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}

      - name: Build and push ingestor image
        id: build-ingestor
        env:
          DOCKER_BUILDKIT: "1"
        run: |
          TREE_SHA="${{ steps.resolve.outputs.tree_sha }}"
          IMAGE="${{ vars.ACR_LOGIN_SERVER }}/api-observatory/ingestor:tree-${TREE_SHA}"
          docker build -t "${IMAGE}" .
          docker push "${IMAGE}"
          echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"

      - name: Check image size (MB)
        shell: bash
        run: |
          set -euo pipefail
          IMAGE="${{ steps.build-ingestor.outputs.image }}"
          docker inspect "${IMAGE}" --format '{{.Size}}' | awk '{mb=$1/1024/1024; printf "Image size: %.0f MB\n", mb}'

  docker-build-dashboard:
    name: "Build and push candidate image (dashboard)"
    runs-on: ubuntu-latest
    needs: [lint, unit-tests, python-deps, action-ref-validation, codeql, integration-tests]
    if: vars.ACR_LOGIN_SERVER != ''
    timeout-minutes: 20
    permissions:
      contents: read
    outputs:
      image_dashboard: ${{ steps.build-dashboard.outputs.image }}
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Resolve tree SHA
        id: resolve
        uses: ./.github/actions/resolve-tree-sha

      - name: Log in to ACR
        uses: docker/login-action@17f7c1809f7467d4244b4ff01b0dbe0b8b9e3d43 # v3.3.0
        with:
          registry: ${{ vars.ACR_LOGIN_SERVER }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}

      - name: Build and push dashboard image
        id: build-dashboard
        env:
          DOCKER_BUILDKIT: "1"
        run: |
          TREE_SHA="${{ steps.resolve.outputs.tree_sha }}"
          IMAGE="${{ vars.ACR_LOGIN_SERVER }}/api-observatory/dashboard:tree-${TREE_SHA}"
          docker build -t "${IMAGE}" -f services/dashboard/Dockerfile services/dashboard
          docker push "${IMAGE}"
          echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"

  docker-scan-security:
    name: "Scan docker image (Trivy - strict)"
    runs-on: ubuntu-latest
    needs: [docker-build, docker-build-dashboard]
    if: vars.ACR_LOGIN_SERVER != ''
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Log in to ACR
        uses: docker/login-action@17f7c1809f7467d4244b4ff01b0dbe0b8b9e3d43 # v3.3.0
        with:
          registry: ${{ vars.ACR_LOGIN_SERVER }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}

      - name: Pull and scan ingestor image
        run: |
          docker pull "${{ needs.docker-build.outputs.image_ingestor }}"
          docker tag "${{ needs.docker-build.outputs.image_ingestor }}" ingestor:scan

      - name: Scan ingestor (Trivy advisory)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: ingestor:scan
          format: table
          ignore-unfixed: true
          scanners: vuln
          vuln-type: os,library
          severity: MEDIUM,HIGH,CRITICAL
          exit-code: "0"

      - name: Pull and scan dashboard image
        run: |
          docker pull "${{ needs.docker-build-dashboard.outputs.image_dashboard }}"
          docker tag "${{ needs.docker-build-dashboard.outputs.image_dashboard }}" dashboard:scan

      - name: Scan dashboard (Trivy advisory)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: dashboard:scan
          format: table
          ignore-unfixed: true
          scanners: vuln
          vuln-type: os,library
          severity: MEDIUM,HIGH,CRITICAL
          exit-code: "0"

  # INTEGRATION TESTS (run after security gates)
  integration-tests:
    name: "Integration tests (postgres + cache)"
    runs-on: ubuntu-latest
    needs: [lint, unit-tests, action-ref-validation]
    timeout-minutes: 15
    services:
      postgres:
        image: pgvector/pgvector:pg17-trixie
        env:
          POSTGRES_DB: api_observatory_test
          POSTGRES_PASSWORD: postgres
          POSTGRES_USER: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 5432:5432
      cache:
        image: redis:7-alpine
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
        ports:
          - 6379:6379
    env:
      DATABASE_URL_TEST: postgresql+asyncpg://postgres:postgres@localhost:5432/api_observatory_test
      CACHE_ENABLED: "true"
      CACHE_URL: redis://localhost:6379/0
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Cache uv
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/uv
          key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
          restore-keys: uv-${{ runner.os }}-

      - name: Install uv
        shell: bash
        run: |
          set -euo pipefail
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Sync dependencies
        run: uv sync --frozen

      - name: Verify service health before tests
        shell: bash
        run: |
          set -euo pipefail

          for _ in {1..30}; do
            if docker exec "${{ job.services.postgres.id }}" pg_isready -U postgres >/dev/null 2>&1; then
              break
            fi
            sleep 2
          done
          docker exec "${{ job.services.postgres.id }}" pg_isready -U postgres

          for _ in {1..30}; do
            if docker exec "${{ job.services.cache.id }}" redis-cli ping | grep -q PONG; then
              break
            fi
            sleep 2
          done
          docker exec "${{ job.services.cache.id }}" redis-cli ping

      - name: Capture dependency diagnostics
        if: failure()
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p ci-diagnostics
          docker ps -a > ci-diagnostics/docker-ps.txt
          docker logs "${{ job.services.postgres.id }}" > ci-diagnostics/postgres.log 2>&1 || true
          docker logs "${{ job.services.cache.id }}" > ci-diagnostics/cache.log 2>&1 || true

      - name: Upload dependency diagnostics
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ci-dependency-diagnostics
          path: ci-diagnostics

      - name: Run integration tests with coverage gate
        run: |
          uv run pytest -q -m integration \
            --cov=services/ingestor \
            --cov=libs \
            --cov-report=term-missing \
            --cov-fail-under=40

  # GATE: ALL CHECKS MUST PASS
  ci-gate:
    name: "CI gate (all checks passed)"
    runs-on: ubuntu-latest
    needs:
      - lint
      - action-ref-validation
      - unit-tests
      - python-deps
      - docker-build
      - docker-build-dashboard
      - docker-scan-security
      - codeql
      - integration-tests
    if: always()
    permissions:
      contents: read
    steps:
      - name: Verify all required checks passed
        run: |
          failed=0
          [[ "${{ needs.lint.result }}" == "success" ]] || { echo "❌ Lint"; failed=1; }
          [[ "${{ needs.action-ref-validation.result }}" == "success" ]] || { echo "❌ Action refs"; failed=1; }
          [[ "${{ needs.unit-tests.result }}" == "success" ]] || { echo "❌ Unit tests"; failed=1; }
          [[ "${{ needs.python-deps.result }}" == "success" ]] || { echo "❌ Python deps"; failed=1; }
          # Docker build/push/scan require ACR_LOGIN_SERVER; they report "skipped"
          # (not failure) until Azure Container Registry credentials are provisioned.
          [[ "${{ needs.docker-build.result }}" =~ ^(success|skipped)$ ]] || { echo "❌ Docker build (ingestor)"; failed=1; }
          [[ "${{ needs.docker-build-dashboard.result }}" =~ ^(success|skipped)$ ]] || { echo "❌ Docker build (dashboard)"; failed=1; }
          [[ "${{ needs.docker-scan-security.result }}" =~ ^(success|skipped)$ ]] || { echo "❌ Docker scan"; failed=1; }
          [[ "${{ needs.codeql.result }}" == "success" ]] || { echo "❌ CodeQL"; failed=1; }
          [[ "${{ needs.integration-tests.result }}" == "success" ]] || { echo "❌ Integration tests"; failed=1; }
          # Advisory scan: report status but do not block
          [[ "${{ needs.docker-scan-security.result }}" =~ ^(success|skipped)$ ]] || echo "⚠️  Docker advisory scan: ${{ needs.docker-scan-security.result }}"
          [ $failed -eq 0 ] && echo "✅ All CI checks passed"
          exit $failed

      - name: Publish summary
        if: always()
        shell: bash
        run: |
          cat >> "$GITHUB_STEP_SUMMARY" <<'EOF'
          ## CI Workflow Summary
          | Check | Result |
          |-------|--------|
          | Lint | ${{ needs.lint.result }} |
          | Action Refs | ${{ needs.action-ref-validation.result }} |
          | Unit Tests | ${{ needs.unit-tests.result }} |
          | Python Deps | ${{ needs.python-deps.result }} |
          | Docker Build (ingestor) | ${{ needs.docker-build.result }} |
          | Docker Build (dashboard) | ${{ needs.docker-build-dashboard.result }} |
          | Docker Security | ${{ needs.docker-scan-security.result }} |
          | Docker Advisory | ${{ needs.docker-scan-security.result }} |
          | CodeQL | ${{ needs.codeql.result }} |
          | Integration Tests | ${{ needs.integration-tests.result }} |
          EOF