pr.yaml

name: PR checks

on:
  pull_request:
    branches: [main]

concurrency:
  group: pr-checks-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  check-fork:
    runs-on: ubuntu-latest
    steps:
      - name: Check if PR is from fork
        run: |
          if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
            echo "::error::PRs must be from branches in PolicyEngine/policyengine-us-data, not forks."
            exit 1
          fi

  check-lock-freshness:
    runs-on: ubuntu-latest
    needs: check-fork
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - name: Check lock file is up-to-date
        run: |
          uv lock --locked || {
            echo "::error::uv.lock is outdated. Run 'uv lock' and commit."
            exit 1
          }

  policyengine-us-freshness:
    name: PolicyEngine US freshness
    runs-on: ubuntu-latest
    needs: check-fork
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - name: Require current PolicyEngine US dependency
        run: python .github/scripts/check_policyengine_us_dependency.py --mode fail

  lint:
    runs-on: ubuntu-latest
    needs: check-fork
    steps:
      - uses: actions/checkout@v6
      - run: pip install ruff>=0.9.0
      - run: ruff format --check .

  quality-guards:
    name: Quality guards
    runs-on: ubuntu-latest
    needs: check-fork
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - name: Run quality guards
        run: uv run --no-sync --with pyyaml python scripts/run_quality_guards.py

  pipeline-docs-build:
    name: Pipeline docs build
    runs-on: ubuntu-latest
    needs: check-fork
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - name: Build generated pipeline docs
        run: |
          out_dir="$(mktemp -d)"
          uv run --no-sync --with pyyaml python scripts/extract_pipeline_docs.py \
            --json "$out_dir/pipeline_map.json" \
            --api-json "$out_dir/pipeline_api.json" \
            --markdown "$out_dir/pipeline-map.md"
          python -m json.tool "$out_dir/pipeline_map.json" >/dev/null
          python -m json.tool "$out_dir/pipeline_api.json" >/dev/null
          test -s "$out_dir/pipeline-map.md"

  check-changelog:
    runs-on: ubuntu-latest
    needs: check-fork
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - run: uv sync --dev
      - name: Check for changelog fragment
        run: uv run towncrier check --compare-with origin/main

  unit-tests:
    runs-on: ubuntu-latest
    needs: [check-fork, lint]
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - run: uv sync --dev
      - name: Run unit tests with coverage
        env:
          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
        run: >
          uv run pytest tests/unit/
          --cov=policyengine_us_data
          --cov-report=xml
          -v
      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@v6
        with:
          files: coverage.xml
          flags: unit
          fail_ci_if_error: false
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  bundle-release-manifest-contract:
    name: Validate bundle release manifest contract
    runs-on: ubuntu-latest
    needs: [check-fork, lint]
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - run: uv sync --dev
      - name: Install bundle contract validator
        run: >
          uv pip install
          "policyengine-bundles @ git+https://github.com/PolicyEngine/policyengine-bundles@e15d653b40926cfb39ddef9b445d1b0022da1b85"
      - name: Validate release manifest shape
        run: >
          uv run pytest
          tests/unit/test_release_manifest.py::test_build_release_manifest_validates_against_bundle_contract
          -v

  target-integration-tests:
    runs-on: ubuntu-latest
    needs: [check-fork, lint, unit-tests, smoke-test, decide-test-scope]
    if: needs.decide-test-scope.outputs.run_integration == 'true'
    name: Integration tests
    env:
      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
      MODAL_PROXY_TOKEN_ID: ${{ secrets.MODAL_PROXY_TOKEN_ID }}
      MODAL_PROXY_TOKEN_SECRET: ${{ secrets.MODAL_PROXY_TOKEN_SECRET }}
      HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
      GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
      # Modal PR environments cannot reliably receive secrets with the CI token.
      # Deploy isolated PR apps and volumes into main, where required secrets
      # already exist, then stop/delete the PR resources in cleanup steps.
      MODAL_ENVIRONMENT: main
      MODAL_APP_NAME: us-data-pipeline-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      MODAL_LOCAL_AREA_APP_NAME: us-data-local-area-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      MODAL_H5_TEST_HARNESS_APP_NAME: us-data-h5-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_PIPELINE_APP_NAME: us-data-pipeline-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_MODAL_APP_NAME: us-data-pipeline-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_LOCAL_AREA_APP_NAME: us-data-local-area-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_H5_HARNESS_APP_NAME: us-data-h5-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_PIPELINE_VOLUME_NAME: pipeline-artifacts-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_STAGING_VOLUME_NAME: local-area-staging-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
      US_DATA_CHECKPOINT_VOLUME_NAME: data-build-checkpoints-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: astral-sh/setup-uv@v8.1.0
      - run: uv sync --dev
      - name: Install integration test deps
        run: uv pip install modal pytest numpy pandas
      - name: Deploy PR Modal pipeline app
        run: uv run modal deploy --env="${MODAL_ENVIRONMENT}" modal_app/pipeline.py
      - name: Deploy PR Modal local-area app
        run: uv run modal deploy --env="${MODAL_ENVIRONMENT}" modal_app/local_area.py
      - name: Deploy PR Modal H5 test harness
        run: uv run modal deploy --env="${MODAL_ENVIRONMENT}" modal_app/h5_test_harness.py
      - name: Run integration tests
        run: >
          uv run pytest
          tests/integration/test_tiny_pipeline_workspace.py
          tests/integration/test_tiny_stage_1_artifacts.py
          tests/integration/test_tiny_stage_2_artifacts.py
          tests/integration/test_tiny_stage_3_artifacts.py
          tests/integration/test_tiny_stage_4_artifacts.py
          tests/integration/test_tiny_stage_5_artifacts.py
          tests/integration/test_tiny_pipeline_e2e.py
          tests/integration/test_tiny_pipeline_h5_e2e.py
          tests/integration/build_outputs/
          tests/integration/test_modal_pipeline_seams.py
          tests/integration/test_tiny_h5_pipeline.py
          tests/integration/test_modal_pipeline_e2e.py
          -v
      - name: Stop PR Modal apps
        if: always()
        run: |
          for app_name in \
            "${MODAL_H5_TEST_HARNESS_APP_NAME}" \
            "${MODAL_LOCAL_AREA_APP_NAME}" \
            "${MODAL_APP_NAME}"
          do
            yes | uv run modal app stop \
              --env="${MODAL_ENVIRONMENT}" \
              "${app_name}" || true
          done
      - name: Delete PR Modal volumes
        if: always()
        run: |
          for volume_name in \
            "${US_DATA_STAGING_VOLUME_NAME}" \
            "${US_DATA_PIPELINE_VOLUME_NAME}" \
            "${US_DATA_CHECKPOINT_VOLUME_NAME}"
          do
            uv run modal volume delete \
              --env="${MODAL_ENVIRONMENT}" \
              --allow-missing \
              --yes \
              "${volume_name}" || true
          done

  smoke-test:
    runs-on: ubuntu-latest
    needs: [check-fork, lint]
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - run: python -m pip install .
      - run: python .github/scripts/check_python_import.py policyengine_us_data
      - run: python .github/scripts/check_python_import.py policyengine_core.data:Dataset

  docs-build:
    name: Documentation build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
      - uses: actions/setup-node@v6
        with:
          node-version: "24"
      - uses: astral-sh/setup-uv@v8.1.0
      - name: Test documentation builds
        run: uv run --no-sync --with "mystmd>=1.7.0" make documentation

  decide-test-scope:
    runs-on: ubuntu-latest
    needs: check-fork
    outputs:
      run_integration: ${{ steps.check.outputs.run_integration }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - name: Check changed files for integration scope
        id: check
        run: |
          CHANGED=$(git diff --name-only origin/main...HEAD)
          if echo "$CHANGED" | grep -qE '^(\.github/scripts/|\.github/workflows/pr\.yaml|modal_app/|policyengine_us_data/|tests/integration/|tests/support/|pyproject\.toml|uv\.lock)'; then
            echo "run_integration=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_integration=false" >> "$GITHUB_OUTPUT"
          fi