Skip to content

Commit 7c8f5ce

Browse files
authored
Keep policyengine-us dependency fresh for data builds (#990)
1 parent 17c1283 commit 7c8f5ce

9 files changed

Lines changed: 391 additions & 4 deletions

File tree

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
"""Check whether the locked PolicyEngine US dependency is current."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
import json
7+
import os
8+
from itertools import zip_longest
9+
from pathlib import Path
10+
import re
11+
import sys
12+
import tomllib
13+
from urllib.error import URLError
14+
from urllib.request import urlopen
15+
16+
17+
REPO_ROOT = Path(__file__).resolve().parents[2]
18+
PYPI_JSON_TIMEOUT_SECONDS = 20
19+
POLICYENGINE_US = "policyengine-us"
20+
STALE_LOCK_PREFIX = "uv.lock has policyengine-us "
21+
22+
23+
def _annotation(level: str, message: str) -> str:
24+
escaped = message.replace("%", "%25").replace("\n", "%0A").replace("\r", "%0D")
25+
return f"::{level} title=PolicyEngine US dependency::{escaped}"
26+
27+
28+
def _version_key(version: str) -> tuple[int, ...]:
29+
release = version.split("+", 1)[0].split("-", 1)[0]
30+
if not re.fullmatch(r"\d+(?:\.\d+)*", release):
31+
raise ValueError(f"Unsupported version format: {version}")
32+
return tuple(int(part) for part in release.split("."))
33+
34+
35+
def _compare_versions(left: str, right: str) -> int:
36+
for left_part, right_part in zip_longest(
37+
_version_key(left), _version_key(right), fillvalue=0
38+
):
39+
if left_part < right_part:
40+
return -1
41+
if left_part > right_part:
42+
return 1
43+
return 0
44+
45+
46+
def _locked_policyengine_us(root: Path) -> tuple[str, dict[str, object]]:
47+
with (root / "uv.lock").open("rb") as file:
48+
lock = tomllib.load(file)
49+
50+
for package in lock.get("package", []):
51+
if package.get("name") == POLICYENGINE_US:
52+
version = package.get("version")
53+
if not isinstance(version, str):
54+
raise ValueError("uv.lock entry for policyengine-us has no version")
55+
source = package.get("source", {})
56+
return version, source if isinstance(source, dict) else {}
57+
58+
raise ValueError("uv.lock does not contain policyengine-us")
59+
60+
61+
def _project_policyengine_us_dependency(root: Path) -> str:
62+
with (root / "pyproject.toml").open("rb") as file:
63+
pyproject = tomllib.load(file)
64+
65+
dependencies = pyproject.get("project", {}).get("dependencies", [])
66+
for dependency in dependencies:
67+
if isinstance(dependency, str) and dependency.startswith(POLICYENGINE_US):
68+
return dependency
69+
70+
raise ValueError("pyproject.toml does not declare policyengine-us")
71+
72+
73+
def _latest_pypi_version() -> str:
74+
with urlopen(
75+
f"https://pypi.org/pypi/{POLICYENGINE_US}/json",
76+
timeout=PYPI_JSON_TIMEOUT_SECONDS,
77+
) as response:
78+
payload = json.load(response)
79+
80+
version = payload.get("info", {}).get("version")
81+
if not isinstance(version, str) or not version:
82+
raise ValueError("PyPI response does not include info.version")
83+
return version
84+
85+
86+
def check_dependency(root: Path, latest_version: str | None = None) -> list[str]:
87+
locked_version, source = _locked_policyengine_us(root)
88+
project_dependency = _project_policyengine_us_dependency(root)
89+
90+
violations: list[str] = []
91+
if (
92+
latest_version is not None
93+
and _compare_versions(locked_version, latest_version) < 0
94+
):
95+
violations.append(
96+
f"{STALE_LOCK_PREFIX}{locked_version}, but PyPI latest is "
97+
f"{latest_version}. Update pyproject.toml and run 'uv lock', or "
98+
"explicitly document why the older model is required."
99+
)
100+
101+
expected_dependency = f"{POLICYENGINE_US}=={locked_version}"
102+
if project_dependency != expected_dependency:
103+
violations.append(
104+
f"pyproject.toml must pin {expected_dependency} to match uv.lock; "
105+
f"found {project_dependency!r}."
106+
)
107+
108+
if "git" in source:
109+
violations.append(
110+
"uv.lock resolves policyengine-us from a Git ref. Prefer an exact "
111+
f"PyPI release pin once policyengine-us {locked_version} is published."
112+
)
113+
114+
if "@" in project_dependency and "git+" in project_dependency:
115+
violations.append(
116+
"pyproject.toml pins policyengine-us to a Git ref. Prefer an exact "
117+
"PyPI release pin for production data builds."
118+
)
119+
120+
return violations
121+
122+
123+
def main() -> int:
124+
parser = argparse.ArgumentParser()
125+
parser.add_argument(
126+
"--mode",
127+
choices=("warn", "fail"),
128+
default="warn",
129+
help="Whether stale dependency findings should fail the command.",
130+
)
131+
parser.add_argument(
132+
"--allow-stale",
133+
action="store_true",
134+
help="Report stale dependency findings but exit successfully.",
135+
)
136+
args = parser.parse_args()
137+
allow_stale = args.allow_stale or os.environ.get(
138+
"POLICYENGINE_US_ALLOW_STALE", ""
139+
).lower() in {"1", "true", "yes"}
140+
141+
latest_version = None
142+
try:
143+
latest_version = _latest_pypi_version()
144+
except (OSError, URLError, ValueError) as exc:
145+
message = (
146+
"Unable to fetch the latest policyengine-us version from PyPI; "
147+
f"continuing with local dependency checks only: {exc}"
148+
)
149+
print(_annotation("warning", message))
150+
151+
try:
152+
violations = check_dependency(REPO_ROOT, latest_version=latest_version)
153+
except (OSError, ValueError) as exc:
154+
message = f"Unable to check policyengine-us freshness: {exc}"
155+
if args.mode == "fail":
156+
print(_annotation("error", message), file=sys.stderr)
157+
return 1
158+
print(_annotation("warning", message))
159+
return 0
160+
161+
if not violations:
162+
locked_version, _source = _locked_policyengine_us(REPO_ROOT)
163+
print(f"policyengine-us dependency is current at {locked_version}.")
164+
return 0
165+
166+
has_blocking_violation = False
167+
allowed_stale_version = False
168+
for violation in violations:
169+
stale_version_violation = violation.startswith(STALE_LOCK_PREFIX)
170+
allowed_by_override = allow_stale and stale_version_violation
171+
level = "warning" if args.mode == "warn" or allowed_by_override else "error"
172+
print(_annotation(level, violation))
173+
if args.mode == "fail" and not allowed_by_override:
174+
has_blocking_violation = True
175+
if allowed_by_override:
176+
allowed_stale_version = True
177+
178+
if allowed_stale_version:
179+
print(
180+
_annotation(
181+
"warning",
182+
"POLICYENGINE_US_ALLOW_STALE is set; continuing despite "
183+
"policyengine-us lagging the latest PyPI release.",
184+
)
185+
)
186+
187+
if has_blocking_violation:
188+
return 1
189+
if allowed_stale_version:
190+
return 0
191+
192+
return 1 if args.mode == "fail" else 0
193+
194+
195+
if __name__ == "__main__":
196+
raise SystemExit(main())

.github/workflows/local_area_publish.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
required: false
2323
default: false
2424
type: boolean
25+
allow_stale_policyengine_us:
26+
description: 'Allow production build when policyengine-us lags the latest PyPI release'
27+
required: false
28+
default: false
29+
type: boolean
2530

2631
# Trigger strategy:
2732
# 1. Automatic: Code changes to calibration/ pushed to main
@@ -50,6 +55,11 @@ jobs:
5055
- name: Install Modal CLI
5156
run: pip install modal
5257

58+
- name: Require current PolicyEngine US dependency
59+
env:
60+
POLICYENGINE_US_ALLOW_STALE: ${{ inputs.allow_stale_policyengine_us }}
61+
run: python .github/scripts/check_policyengine_us_dependency.py --mode fail
62+
5363
- name: Run local area build and stage on Modal
5464
run: |
5565
NUM_WORKERS="${{ github.event.inputs.num_workers || '8' }}"

.github/workflows/long_run_projection.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ on:
108108
required: false
109109
default: ""
110110
type: string
111+
allow_stale_policyengine_us:
112+
description: "Allow production build when policyengine-us lags the latest PyPI release"
113+
required: false
114+
default: false
115+
type: boolean
111116

112117
concurrency:
113118
group: long-run-projection-${{ github.run_id }}-${{ github.run_attempt }}
@@ -139,6 +144,11 @@ jobs:
139144
echo "CHECKED_OUT_SHA=${checked_out_sha}" >> "$GITHUB_ENV"
140145
GITHUB_SHA="${checked_out_sha}" python .github/scripts/resolve_run_context.py
141146
147+
- name: Require current PolicyEngine US dependency
148+
env:
149+
POLICYENGINE_US_ALLOW_STALE: ${{ inputs.allow_stale_policyengine_us }}
150+
run: python .github/scripts/check_policyengine_us_dependency.py --mode fail
151+
142152
- name: Install dependencies
143153
run: uv sync --dev
144154

.github/workflows/pipeline.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ on:
6767
description: "Number of Modal workers for parallel matrix build"
6868
default: "50"
6969
type: string
70+
allow_stale_policyengine_us:
71+
description: "Allow production build when policyengine-us lags the latest PyPI release"
72+
default: false
73+
type: boolean
7074

7175
concurrency:
7276
group: pipeline-${{ github.run_id }}-${{ github.run_attempt }}
@@ -99,6 +103,11 @@ jobs:
99103
RELEASE_BUMP: ${{ inputs.release_bump || '' }}
100104
run: python .github/scripts/resolve_run_context.py
101105

106+
- name: Require current PolicyEngine US dependency
107+
env:
108+
POLICYENGINE_US_ALLOW_STALE: ${{ inputs.allow_stale_policyengine_us }}
109+
run: python .github/scripts/check_policyengine_us_dependency.py --mode fail
110+
102111
- name: Deploy and launch pipeline on Modal
103112
env:
104113
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}

.github/workflows/pr.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,18 @@ jobs:
3535
exit 1
3636
}
3737
38+
policyengine-us-freshness:
39+
name: PolicyEngine US freshness
40+
runs-on: ubuntu-latest
41+
needs: check-fork
42+
steps:
43+
- uses: actions/checkout@v6
44+
- uses: actions/setup-python@v6
45+
with:
46+
python-version: "3.14"
47+
- name: Warn if policyengine-us is stale
48+
run: python .github/scripts/check_policyengine_us_dependency.py --mode warn
49+
3850
lint:
3951
runs-on: ubuntu-latest
4052
needs: check-fork
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Use the released PolicyEngine US 1.691.11 model for data builds and warn when the locked model lags PyPI.

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ classifiers = [
2222
"Programming Language :: Python :: 3.14",
2323
]
2424
dependencies = [
25-
"policyengine-us @ git+https://github.com/PolicyEngine/policyengine-us@4fd79e6608bc2dac3a7fde0be37191cb4870bd85",
25+
"policyengine-us==1.691.11",
2626
# policyengine-core 3.26.1 is the current 3.26.x runtime and includes the fix for
2727
# PolicyEngine/policyengine-core#482 (user-set ETERNITY inputs lost
2828
# after _invalidate_all_caches) and is required by policyengine-us 1.682.1+.

0 commit comments

Comments
 (0)