Add Stage 2 package identity specs #1024
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR checks | |
| on: | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: pr-checks-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-fork: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check if PR is from fork | |
| run: | | |
| if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then | |
| echo "::error::PRs must be from branches in PolicyEngine/policyengine-us-data, not forks." | |
| exit 1 | |
| fi | |
| check-lock-freshness: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - name: Check lock file is up-to-date | |
| run: | | |
| uv lock --locked || { | |
| echo "::error::uv.lock is outdated. Run 'uv lock' and commit." | |
| exit 1 | |
| } | |
| policyengine-us-freshness: | |
| name: PolicyEngine US freshness | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - name: Require current PolicyEngine US dependency | |
| run: python .github/scripts/check_policyengine_us_dependency.py --mode fail | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - run: pip install ruff>=0.9.0 | |
| - run: ruff format --check . | |
| quality-guards: | |
| name: Quality guards | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - name: Run quality guards | |
| run: uv run --no-sync --with pyyaml python scripts/run_quality_guards.py | |
| pipeline-docs-build: | |
| name: Pipeline docs build | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - name: Build generated pipeline docs | |
| run: | | |
| out_dir="$(mktemp -d)" | |
| uv run --no-sync --with pyyaml python scripts/extract_pipeline_docs.py \ | |
| --json "$out_dir/pipeline_map.json" \ | |
| --api-json "$out_dir/pipeline_api.json" \ | |
| --markdown "$out_dir/pipeline-map.md" | |
| python -m json.tool "$out_dir/pipeline_map.json" >/dev/null | |
| python -m json.tool "$out_dir/pipeline_api.json" >/dev/null | |
| test -s "$out_dir/pipeline-map.md" | |
| check-changelog: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - run: uv sync --dev | |
| - name: Check for changelog fragment | |
| run: uv run towncrier check --compare-with origin/main | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - run: uv sync --dev | |
| - name: Run unit tests with coverage | |
| env: | |
| HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} | |
| run: > | |
| uv run pytest tests/unit/ | |
| --cov=policyengine_us_data | |
| --cov-report=xml | |
| -v | |
| - name: Upload coverage to Codecov | |
| if: always() | |
| uses: codecov/codecov-action@v6 | |
| with: | |
| files: coverage.xml | |
| flags: unit | |
| fail_ci_if_error: false | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| bundle-release-manifest-contract: | |
| name: Validate bundle release manifest contract | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - run: uv sync --dev | |
| - name: Install bundle contract validator | |
| run: > | |
| uv pip install | |
| "policyengine-bundles @ git+https://github.com/PolicyEngine/policyengine-bundles@e15d653b40926cfb39ddef9b445d1b0022da1b85" | |
| - name: Validate release manifest shape | |
| run: > | |
| uv run pytest | |
| tests/unit/test_release_manifest.py::test_build_release_manifest_validates_against_bundle_contract | |
| -v | |
| target-integration-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint, unit-tests, smoke-test, decide-test-scope] | |
| if: needs.decide-test-scope.outputs.run_integration == 'true' | |
| name: Integration tests | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| MODAL_PROXY_TOKEN_ID: ${{ secrets.MODAL_PROXY_TOKEN_ID }} | |
| MODAL_PROXY_TOKEN_SECRET: ${{ secrets.MODAL_PROXY_TOKEN_SECRET }} | |
| HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} | |
| # Modal PR environments cannot reliably receive secrets with the CI token. | |
| # Deploy isolated PR apps and volumes into main, where required secrets | |
| # already exist, then stop/delete the PR resources in cleanup steps. | |
| MODAL_ENVIRONMENT: main | |
| MODAL_APP_NAME: us-data-pipeline-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| MODAL_LOCAL_AREA_APP_NAME: us-data-local-area-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| MODAL_H5_TEST_HARNESS_APP_NAME: us-data-h5-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_PIPELINE_APP_NAME: us-data-pipeline-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_MODAL_APP_NAME: us-data-pipeline-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_LOCAL_AREA_APP_NAME: us-data-local-area-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_H5_HARNESS_APP_NAME: us-data-h5-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_PIPELINE_VOLUME_NAME: pipeline-artifacts-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_STAGING_VOLUME_NAME: local-area-staging-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| US_DATA_CHECKPOINT_VOLUME_NAME: data-build-checkpoints-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - run: uv sync --dev | |
| - name: Install integration test deps | |
| run: uv pip install modal pytest numpy pandas | |
| - name: Deploy PR Modal pipeline app | |
| run: uv run modal deploy --env="${MODAL_ENVIRONMENT}" modal_app/pipeline.py | |
| - name: Deploy PR Modal local-area app | |
| run: uv run modal deploy --env="${MODAL_ENVIRONMENT}" modal_app/local_area.py | |
| - name: Deploy PR Modal H5 test harness | |
| run: uv run modal deploy --env="${MODAL_ENVIRONMENT}" modal_app/h5_test_harness.py | |
| - name: Run integration tests | |
| run: > | |
| uv run pytest | |
| tests/integration/test_tiny_pipeline_workspace.py | |
| tests/integration/test_tiny_stage_1_artifacts.py | |
| tests/integration/test_tiny_stage_2_artifacts.py | |
| tests/integration/test_tiny_stage_3_artifacts.py | |
| tests/integration/test_tiny_stage_4_artifacts.py | |
| tests/integration/test_tiny_stage_5_artifacts.py | |
| tests/integration/test_tiny_pipeline_e2e.py | |
| tests/integration/test_tiny_pipeline_h5_e2e.py | |
| tests/integration/build_outputs/ | |
| tests/integration/test_modal_pipeline_seams.py | |
| tests/integration/test_tiny_h5_pipeline.py | |
| tests/integration/test_modal_pipeline_e2e.py | |
| -v | |
| - name: Stop PR Modal apps | |
| if: always() | |
| run: | | |
| for app_name in \ | |
| "${MODAL_H5_TEST_HARNESS_APP_NAME}" \ | |
| "${MODAL_LOCAL_AREA_APP_NAME}" \ | |
| "${MODAL_APP_NAME}" | |
| do | |
| yes | uv run modal app stop \ | |
| --env="${MODAL_ENVIRONMENT}" \ | |
| "${app_name}" || true | |
| done | |
| - name: Delete PR Modal volumes | |
| if: always() | |
| run: | | |
| for volume_name in \ | |
| "${US_DATA_STAGING_VOLUME_NAME}" \ | |
| "${US_DATA_PIPELINE_VOLUME_NAME}" \ | |
| "${US_DATA_CHECKPOINT_VOLUME_NAME}" | |
| do | |
| uv run modal volume delete \ | |
| --env="${MODAL_ENVIRONMENT}" \ | |
| --allow-missing \ | |
| --yes \ | |
| "${volume_name}" || true | |
| done | |
| smoke-test: | |
| runs-on: ubuntu-latest | |
| needs: [check-fork, lint] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - run: python -m pip install . | |
| - run: python .github/scripts/check_python_import.py policyengine_us_data | |
| - run: python .github/scripts/check_python_import.py policyengine_core.data:Dataset | |
| docs-build: | |
| name: Documentation build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| - uses: astral-sh/setup-uv@v8.1.0 | |
| - name: Test documentation builds | |
| run: uv run --no-sync --with "mystmd>=1.7.0" make documentation | |
| decide-test-scope: | |
| runs-on: ubuntu-latest | |
| needs: check-fork | |
| outputs: | |
| run_integration: ${{ steps.check.outputs.run_integration }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check changed files for integration scope | |
| id: check | |
| run: | | |
| CHANGED=$(git diff --name-only origin/main...HEAD) | |
| if echo "$CHANGED" | grep -qE '^(\.github/scripts/|\.github/workflows/pr\.yaml|modal_app/|policyengine_us_data/|tests/integration/|tests/support/|pyproject\.toml|uv\.lock)'; then | |
| echo "run_integration=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_integration=false" >> "$GITHUB_OUTPUT" | |
| fi |