diff --git a/.github/scripts/modal-deploy-versioned.sh b/.github/scripts/modal-deploy-versioned.sh index 614a7d87..62c2f4e5 100755 --- a/.github/scripts/modal-deploy-versioned.sh +++ b/.github/scripts/modal-deploy-versioned.sh @@ -1,7 +1,7 @@ #!/bin/bash # Deploy versioned simulation app to Modal # Usage: ./modal-deploy-versioned.sh -# Required env vars: POLICYENGINE_US_VERSION, POLICYENGINE_UK_VERSION +# Required env vars: POLICYENGINE_VERSION, POLICYENGINE_US_VERSION, POLICYENGINE_UK_VERSION # # Deploys a versioned app named policyengine-us{X}-uk{Y} and updates # the Modal Dict version registries so Cloud Run can route to it. @@ -14,6 +14,7 @@ MODAL_ENV="${1:?Modal environment required (staging or main)}" # Validate required env vars : "${POLICYENGINE_US_VERSION:?POLICYENGINE_US_VERSION must be set}" : "${POLICYENGINE_UK_VERSION:?POLICYENGINE_UK_VERSION must be set}" +: "${POLICYENGINE_VERSION:?POLICYENGINE_VERSION must be set}" # Generate versioned app name (dots replaced with dashes) US_VERSION_SAFE="${POLICYENGINE_US_VERSION//./-}" @@ -24,6 +25,7 @@ echo "========================================" echo "Deploying versioned Modal simulation app" echo " Environment: $MODAL_ENV" echo " App name: $APP_NAME" +echo " policyengine.py: ${POLICYENGINE_VERSION}" echo " US version: ${POLICYENGINE_US_VERSION}" echo " UK version: ${POLICYENGINE_UK_VERSION}" echo "========================================" diff --git a/.github/scripts/modal-extract-versions.sh b/.github/scripts/modal-extract-versions.sh index ebf63257..5b4cb8a2 100755 --- a/.github/scripts/modal-extract-versions.sh +++ b/.github/scripts/modal-extract-versions.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Extract policyengine-us and policyengine-uk versions from uv.lock +# Extract policyengine, policyengine-us, and policyengine-uk versions from uv.lock # Usage: ./modal-extract-versions.sh [project-dir] -# Outputs: Sets us_version and uk_version in GITHUB_OUTPUT +# Outputs: Sets policyengine_version, us_version, and uk_version in GITHUB_OUTPUT set -euo pipefail @@ -9,16 +9,19 @@ PROJECT_DIR="${1:-.}" cd "$PROJECT_DIR" +POLICYENGINE_VERSION=$(grep -A1 'name = "policyengine"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') US_VERSION=$(grep -A1 'name = "policyengine-us"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') UK_VERSION=$(grep -A1 'name = "policyengine-uk"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') -if [ -z "$US_VERSION" ] || [ -z "$UK_VERSION" ]; then +if [ -z "$POLICYENGINE_VERSION" ] || [ -z "$US_VERSION" ] || [ -z "$UK_VERSION" ]; then echo "ERROR: Could not extract versions from uv.lock" + echo " POLICYENGINE_VERSION=$POLICYENGINE_VERSION" echo " US_VERSION=$US_VERSION" echo " UK_VERSION=$UK_VERSION" exit 1 fi +echo "policyengine_version=$POLICYENGINE_VERSION" >> "$GITHUB_OUTPUT" echo "us_version=$US_VERSION" >> "$GITHUB_OUTPUT" echo "uk_version=$UK_VERSION" >> "$GITHUB_OUTPUT" -echo "Extracted versions: policyengine-us=$US_VERSION, policyengine-uk=$UK_VERSION" +echo "Extracted versions: policyengine=$POLICYENGINE_VERSION, policyengine-us=$US_VERSION, policyengine-uk=$UK_VERSION" diff --git a/.github/workflows/db-reseed.yml b/.github/workflows/db-reseed.yml new file mode 100644 index 00000000..ab988324 --- /dev/null +++ b/.github/workflows/db-reseed.yml @@ -0,0 +1,102 @@ +name: Reseed database + +on: + workflow_dispatch: + inputs: + target: + description: "Target database" + required: true + type: choice + options: + - staging + - production + preset: + description: "Seeding preset" + required: true + default: "full" + type: choice + options: + - full + - lite + - minimal + - uk-lite + - uk-minimal + - us-lite + - us-minimal + - testing + confirm: + description: "Type 'reseed-staging' or 'reseed-prod' to confirm" + required: true + type: string + pull_request: + paths: + - ".github/workflows/db-reseed.yml" + +jobs: + validate: + name: Validate workflow + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Validate workflow syntax + run: | + bash <(curl -sSL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + ./actionlint -color + + reseed-db: + name: Reseed ${{ inputs.target }} database + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + environment: ${{ inputs.target }} + + steps: + - name: Verify confirmation + run: | + EXPECTED="reseed-staging" + if [ "${{ inputs.target }}" = "production" ]; then + EXPECTED="reseed-prod" + fi + if [ "${{ inputs.confirm }}" != "$EXPECTED" ]; then + echo "Confirmation failed. You must type '$EXPECTED' to proceed." + exit 1 + fi + echo "Confirmation verified for ${{ inputs.target }}" + + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + save-cache: false + + - name: Setup Python + run: uv python install 3.13 + + - name: Sync dependencies + run: uv sync + + - name: Reseed database + env: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} + SUPABASE_SECRET_KEY: ${{ secrets.SUPABASE_SECRET_KEY }} + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + STORAGE_BUCKET: ${{ vars.STORAGE_BUCKET }} + LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }} + LOGFIRE_ENVIRONMENT: ${{ inputs.target }} + run: | + echo "Reseeding ${{ inputs.target }} database with preset '${{ inputs.preset }}'..." + uv run python scripts/seed.py --preset="${{ inputs.preset }}" + + - name: Summary + run: | + echo "Database reseed complete." + echo "Target: ${{ inputs.target }}" + echo "Preset: ${{ inputs.preset }}" + echo "Triggered by: ${{ github.actor }}" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5c3f12b6..28d4f40b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -131,14 +131,14 @@ jobs: uses: google-github-actions/setup-gcloud@v3 - name: Configure Docker for Artifact Registry - run: gcloud auth configure-docker ${{ vars.GCP_REGION }}-docker.pkg.dev + run: gcloud auth configure-docker "${{ vars.GCP_REGION }}-docker.pkg.dev" - name: Build and push Docker image run: | - docker build -t $IMAGE_URL:${{ github.sha }} . - docker tag $IMAGE_URL:${{ github.sha }} $IMAGE_URL:latest - docker push $IMAGE_URL:${{ github.sha }} - docker push $IMAGE_URL:latest + docker build -t "${IMAGE_URL}:${{ github.sha }}" . + docker tag "${IMAGE_URL}:${{ github.sha }}" "${IMAGE_URL}:latest" + docker push "${IMAGE_URL}:${{ github.sha }}" + docker push "${IMAGE_URL}:latest" infra: name: Apply infrastructure @@ -262,6 +262,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + POLICYENGINE_VERSION: ${{ steps.versions.outputs.policyengine_version }} POLICYENGINE_US_VERSION: ${{ steps.versions.outputs.us_version }} POLICYENGINE_UK_VERSION: ${{ steps.versions.outputs.uk_version }} run: | @@ -304,7 +305,7 @@ jobs: run: | gcloud run deploy ${{ vars.API_SERVICE_NAME }} \ --region=${{ vars.GCP_REGION }} \ - --image=$IMAGE_URL:${{ github.sha }} \ + --image="${IMAGE_URL}:${{ github.sha }}" \ --tag=staging \ --no-traffic \ --update-env-vars=MODAL_ENVIRONMENT=staging,LOGFIRE_ENVIRONMENT=staging,SUPABASE_URL=${{ secrets.SUPABASE_URL }},SUPABASE_KEY=${{ secrets.SUPABASE_KEY }},SUPABASE_SECRET_KEY=${{ secrets.SUPABASE_SECRET_KEY }},SUPABASE_DB_URL=${{ secrets.SUPABASE_DB_URL }} @@ -400,6 +401,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + POLICYENGINE_VERSION: ${{ steps.prod-versions.outputs.policyengine_version }} POLICYENGINE_US_VERSION: ${{ steps.prod-versions.outputs.us_version }} POLICYENGINE_UK_VERSION: ${{ steps.prod-versions.outputs.uk_version }} run: | @@ -417,6 +419,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + POLICYENGINE_VERSION: ${{ steps.prod-versions.outputs.policyengine_version }} POLICYENGINE_US_VERSION: ${{ steps.prod-versions.outputs.us_version }} POLICYENGINE_UK_VERSION: ${{ steps.prod-versions.outputs.uk_version }} run: | @@ -469,7 +472,7 @@ jobs: run: | gcloud run deploy ${{ vars.API_SERVICE_NAME }} \ --region=${{ vars.GCP_REGION }} \ - --image=$IMAGE_URL:${{ github.sha }} \ + --image="${IMAGE_URL}:${{ github.sha }}" \ --tag=canary \ --no-traffic \ --update-env-vars=MODAL_ENVIRONMENT=main,LOGFIRE_ENVIRONMENT=prod diff --git a/Makefile b/Makefile index 3f375aeb..a8d3929b 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,8 @@ modal-deploy: "SUPABASE_KEY=$$SUPABASE_KEY" \ "STORAGE_BUCKET=$$STORAGE_BUCKET" \ --force - @export POLICYENGINE_US_VERSION=$$(grep -A1 'name = "policyengine-us"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \ + @export POLICYENGINE_VERSION=$$(grep -A1 'name = "policyengine"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \ + export POLICYENGINE_US_VERSION=$$(grep -A1 'name = "policyengine-us"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \ export POLICYENGINE_UK_VERSION=$$(grep -A1 'name = "policyengine-uk"' uv.lock | grep version | head -1 | sed 's/.*"\(.*\)".*/\1/') && \ .github/scripts/modal-deploy-versioned.sh main diff --git a/changelog.d/update-policyengine-runtime.changed b/changelog.d/update-policyengine-runtime.changed new file mode 100644 index 00000000..e7c58889 --- /dev/null +++ b/changelog.d/update-policyengine-runtime.changed @@ -0,0 +1 @@ +Pin the Modal simulation image to the locked policyengine.py version and add a reseed-only database workflow. diff --git a/pyproject.toml b/pyproject.toml index 074b7fca..4acc713e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ dependencies = [ "psycopg2-binary>=2.9.10", "supabase>=2.10.0", "storage3>=0.8.1", - "policyengine>=3.2.3", - "policyengine-uk==2.75.1", - "policyengine-us==1.666.1", + "policyengine==4.4.4", + "policyengine-uk==2.88.14", + "policyengine-us==1.691.3", "pydantic>=2.9.2", "pydantic-settings>=2.6.0", "rich>=13.9.4", diff --git a/src/policyengine_api/modal/images.py b/src/policyengine_api/modal/images.py index de9907af..74265d80 100644 --- a/src/policyengine_api/modal/images.py +++ b/src/policyengine_api/modal/images.py @@ -9,16 +9,17 @@ import modal -US_VERSION = os.environ.get("POLICYENGINE_US_VERSION", "1.592.4") -UK_VERSION = os.environ.get("POLICYENGINE_UK_VERSION", "2.75.1") +POLICYENGINE_VERSION = os.environ.get("POLICYENGINE_VERSION", "4.4.4") +US_VERSION = os.environ.get("POLICYENGINE_US_VERSION", "1.691.3") +UK_VERSION = os.environ.get("POLICYENGINE_UK_VERSION", "2.88.14") base_image = ( modal.Image.debian_slim(python_version="3.13") .apt_install("libhdf5-dev", "git") .pip_install("uv") .run_commands( - "uv pip install --system --upgrade " - "policyengine>=3.2.0 " + f"uv pip install --system --upgrade " + f"policyengine=={POLICYENGINE_VERSION} " "sqlmodel>=0.0.22 " "psycopg2-binary>=2.9.10 " "supabase>=2.10.0 " diff --git a/uv.lock b/uv.lock index 9edb60c8..2f0d796c 100644 --- a/uv.lock +++ b/uv.lock @@ -1900,24 +1900,25 @@ wheels = [ [[package]] name = "policyengine" -version = "3.2.3" +version = "4.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "jsonschema" }, { name = "microdf-python" }, + { name = "packaging" }, { name = "pandas" }, - { name = "plotly" }, { name = "psutil" }, { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f6/73/0617495f360e3ae7e2e146f4bc19434aaa72a469a34043cc8b4ef1066a5e/policyengine-3.2.3.tar.gz", hash = "sha256:cdbc0b1eb60726b2c8ba58838f7950bf0a2ddb5aad960f099e8f70e5ad819d39", size = 243693, upload-time = "2026-03-17T22:02:05.64Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/ce/db6c0a07f5f7c2fd2288cfe423c79bf5a5f2a524c682908c80d926cdebbd/policyengine-4.4.4.tar.gz", hash = "sha256:e73dad3738825cea45cea55c79a9ce2d0551853a8e8b0ccf3c053e184f878d2c", size = 522978, upload-time = "2026-05-13T12:56:37.714Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/66/5616d6df1c88500dd21b9f4f2593bf3e37a74b1b62d06d7056093ccf9768/policyengine-3.2.3-py3-none-any.whl", hash = "sha256:2fdf9f65513f9c8c3235f315617b4cbc6cd7dc2169bd3102e133b83a8133e790", size = 105831, upload-time = "2026-03-17T22:02:04.499Z" }, + { url = "https://files.pythonhosted.org/packages/1f/9c/472703043946fdcbe9feed644cc6651bb082c3fcb339a05617c1335b5ac8/policyengine-4.4.4-py3-none-any.whl", hash = "sha256:5ea219450502f7d9fb19caf3341860abaf18a201b8ec5b8c753cfb848edc0a39", size = 161020, upload-time = "2026-05-13T12:56:36.144Z" }, ] [[package]] name = "policyengine-api-v2" -version = "0.6.4" +version = "0.6.6" source = { editable = "." } dependencies = [ { name = "alembic" }, @@ -1970,9 +1971,9 @@ requires-dist = [ { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.2" }, { name = "logfire", extras = ["fastapi", "httpx", "sqlalchemy"], specifier = ">=0.60.0" }, { name = "modal", specifier = ">=0.68.0" }, - { name = "policyengine", specifier = ">=3.2.3" }, - { name = "policyengine-uk", specifier = "==2.75.1" }, - { name = "policyengine-us", specifier = "==1.666.1" }, + { name = "policyengine", specifier = "==4.4.4" }, + { name = "policyengine-uk", specifier = "==2.88.14" }, + { name = "policyengine-us", specifier = "==1.691.3" }, { name = "psycopg2-binary", specifier = ">=2.9.10" }, { name = "pydantic", specifier = ">=2.9.2" }, { name = "pydantic-settings", specifier = ">=2.6.0" }, @@ -1995,7 +1996,7 @@ dev = [{ name = "pytest-asyncio", specifier = ">=1.3.0" }] [[package]] name = "policyengine-core" -version = "3.25.2" +version = "3.26.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -2015,14 +2016,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d1/ed/117c487e09bc2d5dae1d39baef2d317158433f66004f38f19f6ed82110eb/policyengine_core-3.25.2.tar.gz", hash = "sha256:de80b64b969e3c6b5a3046e29e5b9f2ce56c82f874064f65556fa60c8c423f17", size = 466431, upload-time = "2026-04-21T17:37:40.654Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/56/666e1e708cbd61078989edc943d5389d45123beb7124a5e3180171656ff6/policyengine_core-3.26.1.tar.gz", hash = "sha256:dc4e3007bcd137cbe608042d067cedb889a9b8671db3d08c8e237f1ac3e324b4", size = 472159, upload-time = "2026-05-07T23:46:43.228Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/22/b2297927a2091a22267c112e8d5d5d2b374d0f1ce67f257816fee1388170/policyengine_core-3.25.2-py3-none-any.whl", hash = "sha256:c884961937940e16730fb473d486728ed8c66250dc65df15257ad611e6655b09", size = 231186, upload-time = "2026-04-21T17:37:39.148Z" }, + { url = "https://files.pythonhosted.org/packages/15/2f/9be635fe4dfb2fe65d200c33695f5a96ef98a2921f06ff6d465384b0e551/policyengine_core-3.26.1-py3-none-any.whl", hash = "sha256:185374b3c1fe13dc951637c49a9853211ca61a8a9971eb9cc4c4b07b1477240a", size = 232190, upload-time = "2026-05-07T23:46:41.797Z" }, ] [[package]] name = "policyengine-uk" -version = "2.75.1" +version = "2.88.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2030,14 +2031,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/02/56d6a79e120c882c8aa292fd82f3d90225d938630eb9427dff40bcc416b1/policyengine_uk-2.75.1.tar.gz", hash = "sha256:34478b267bf5c3110a9202535b757fb9c53d3813a86546b31e7b7c33cf9283c9", size = 1102676, upload-time = "2026-03-10T10:54:32.338Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/2b/ab82aa30c5d27176fd9d449ff5ed9708d0080b00912f7dc2efa0af0fd87e/policyengine_uk-2.88.14.tar.gz", hash = "sha256:21a4387ae52dcb5430b6d790edcc321816ed47147a3a0e21ffd482a36834d352", size = 1185947, upload-time = "2026-05-09T04:19:25.284Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/52/28/cb9eff0ad163563a5dcb9667addb3e4a7929696a45b34b90d46a41f0d622/policyengine_uk-2.75.1-py3-none-any.whl", hash = "sha256:77d2552be3e2df1bbcb95a463a99aee22f9c4f8976c58a1d3cde2831c4454d3a", size = 1697838, upload-time = "2026-03-10T10:54:30.57Z" }, + { url = "https://files.pythonhosted.org/packages/a3/fc/276fb639a46bda35523329d8968bcc4089fde9e97fab82722c0ec853c6cc/policyengine_uk-2.88.14-py3-none-any.whl", hash = "sha256:ed10005ba7d0c973c0966ebbf7672853fb3caaa0456b8bf485fb13f8c323d975", size = 1913671, upload-time = "2026-05-09T04:19:23.364Z" }, ] [[package]] name = "policyengine-us" -version = "1.666.1" +version = "1.691.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2047,9 +2048,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/ed/779ae0f296a5652b1a27c0ff9a210d0907cb28887b693fe390e4816bcadb/policyengine_us-1.666.1.tar.gz", hash = "sha256:796fa45584a7e5795595fc31688a2df9841c35b0811dfe83dfa6df250140a2f4", size = 9309724, upload-time = "2026-04-23T22:36:52.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/04/5e74890db670d7eb736f8fcb6070e6866950ba2e1d79e1d74f0dc5cfc58c/policyengine_us-1.691.3.tar.gz", hash = "sha256:3475c0e71ebe3396cfa2d138c62eeaa22a912c301d941418238109cf51a89e86", size = 9493068, upload-time = "2026-05-12T16:54:08.94Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/c1/aa62214930a4ce1c0bc0906f10b19dc5d67128cfedc1b96dbd4330c113b3/policyengine_us-1.666.1-py3-none-any.whl", hash = "sha256:33ef9c7f0c72bd4d17bca922d4ab4f65b42edc4ab41c950137938880ed6de13b", size = 9589718, upload-time = "2026-04-23T22:36:48.87Z" }, + { url = "https://files.pythonhosted.org/packages/2a/03/e21c872664f90dcc99f1fcf29d1da71409c50cf8a7798ff0596ad10d9400/policyengine_us-1.691.3-py3-none-any.whl", hash = "sha256:c5d37aa4442f23d48bd5d587a02876c89d83c6135809f12988cc39bd3a47e8b2", size = 10008634, upload-time = "2026-05-12T16:54:05.956Z" }, ] [[package]]