Update US release bundle (#356)

* Update US release bundle

* Fix data release manifest TRO hashing

Author: MaxGhenis

changelog.d/us-ss-spm-release.changed

@@ -0,0 +1 @@
+Update the bundled US release to policyengine-us 1.691.3 and policyengine-us-data 1.113.1.

pyproject.toml

@@ -46,7 +46,7 @@ uk = [
 ]
 us = [
     "policyengine_core>=3.26.1",
-    "policyengine-us==1.690.7",
+    "policyengine-us==1.691.3",
 ]
 dev = [
     "pytest",
@@ -61,7 +61,7 @@ dev = [
     "ruff>=0.9.0",
     "policyengine_core>=3.26.1",
     "policyengine-uk==2.88.14",
-    "policyengine-us==1.690.7",
+    "policyengine-us==1.691.3",
     "towncrier>=24.8.0",
     "mypy>=1.11.0",
     "pytest-cov>=5.0.0",

src/policyengine/data/release_manifests/us.json

@@ -5,34 +5,34 @@
   "policyengine_version": "4.4.3",
   "model_package": {
     "name": "policyengine-us",
-    "version": "1.690.7",
-    "sha256": "5a7a541efabac98fa069d6845902cf5924c81db67383234b55dcd2b8bfcfc3ca",
-    "wheel_url": "https://files.pythonhosted.org/packages/2a/02/52109bae5f4767237b43bd72ce0bc4edf7925650a788053b2bc168caa5ae/policyengine_us-1.690.7-py3-none-any.whl"
+    "version": "1.691.3",
+    "sha256": "c5d37aa4442f23d48bd5d587a02876c89d83c6135809f12988cc39bd3a47e8b2",
+    "wheel_url": "https://files.pythonhosted.org/packages/2a/03/e21c872664f90dcc99f1fcf29d1da71409c50cf8a7798ff0596ad10d9400/policyengine_us-1.691.3-py3-none-any.whl"
   },
   "data_package": {
     "name": "policyengine-us-data",
-    "version": "1.110.12",
+    "version": "1.113.1",
     "repo_id": "policyengine/policyengine-us-data",
-    "release_manifest_path": "releases/1.110.12/release_manifest.json",
-    "release_manifest_revision": "3aac4505ec10d31efc1b3799a1e6458a15853ecc"
+    "release_manifest_path": "releases/1.113.1/release_manifest.json",
+    "release_manifest_revision": "99e0ec7e784cdba43dd21ff1d80a081599a7a537"
   },
   "certified_data_artifact": {
     "data_package": {
       "name": "policyengine-us-data",
-      "version": "1.110.12"
+      "version": "1.113.1"
     },
-    "build_id": "policyengine-us-data-1.110.12",
+    "build_id": "policyengine-us-data-1.113.1",
     "dataset": "enhanced_cps_2024",
-    "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.110.12",
-    "sha256": "58a6639f7511b8d804701417e2647f0c3a77f51a3d90441037eaf004b1f00761"
+    "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@99e0ec7e784cdba43dd21ff1d80a081599a7a537",
+    "sha256": "0ce549af18753287c097718362b8cd5cdccfc47953acf7f282709d604cf314d2"
   },
   "certification": {
     "compatibility_basis": "exact_build_model_version",
-    "data_build_id": "policyengine-us-data-1.110.12",
-    "built_with_model_version": "1.690.7",
-    "certified_for_model_version": "1.690.7",
+    "data_build_id": "policyengine-us-data-1.113.1",
+    "built_with_model_version": "1.691.3",
+    "certified_for_model_version": "1.691.3",
     "certified_by": "policyengine.py bundled manifest",
-    "data_build_fingerprint": "sha256:9961ed1c5d00943a360724da560eee425eb9f99f91896f053dca74724c46e96e"
+    "data_build_fingerprint": "sha256:d891044ece8ec3338904771c879b98ec11a12f1090c074e5b8cee846825d8056"
   },
   "default_dataset": "enhanced_cps_2024",
   "datasets": {

src/policyengine/data/release_manifests/us.trace.tro.jsonld

@@ -17,7 +17,7 @@
         "schema:name": "PolicyEngine",
         "schema:url": "https://policyengine.org"
       },
-      "schema:dateCreated": "2026-05-11T18:53:05.508006Z",
+      "schema:dateCreated": "2026-05-13T03:56:37.150215Z",
       "schema:description": "TRACE TRO for certified runtime bundle us-4.4.3 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.",
       "schema:name": "policyengine us certified bundle TRO",
       "trov:createdWith": {
@@ -45,23 +45,23 @@
               "trov:hasArtifact": {
                 "@id": "composition/1/artifact/data_release_manifest"
               },
-              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/3aac4505ec10d31efc1b3799a1e6458a15853ecc/releases/1.110.12/release_manifest.json"
+              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/99e0ec7e784cdba43dd21ff1d80a081599a7a537/releases/1.113.1/release_manifest.json"
             },
             {
               "@id": "arrangement/1/location/dataset",
               "@type": "trov:ArtifactLocation",
               "trov:hasArtifact": {
                 "@id": "composition/1/artifact/dataset"
               },
-              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/1.110.12/enhanced_cps_2024.h5"
+              "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/99e0ec7e784cdba43dd21ff1d80a081599a7a537/enhanced_cps_2024.h5"
             },
             {
               "@id": "arrangement/1/location/model_wheel",
               "@type": "trov:ArtifactLocation",
               "trov:hasArtifact": {
                 "@id": "composition/1/artifact/model_wheel"
               },
-              "trov:hasLocation": "https://files.pythonhosted.org/packages/2a/02/52109bae5f4767237b43bd72ce0bc4edf7925650a788053b2bc168caa5ae/policyengine_us-1.690.7-py3-none-any.whl"
+              "trov:hasLocation": "https://files.pythonhosted.org/packages/2a/03/e21c872664f90dcc99f1fcf29d1da71409c50cf8a7798ff0596ad10d9400/policyengine_us-1.691.3-py3-none-any.whl"
             }
           ]
         }
@@ -75,54 +75,51 @@
             "@type": "trov:ResearchArtifact",
             "schema:name": "policyengine.py bundle manifest for us",
             "trov:mimeType": "application/json",
-            "trov:sha256": "41e196a6263b8168d403058029c52ebab795e17024ac9ebef11ff876e36959e2"
+            "trov:sha256": "67d4d6505bed4af9bf2ec575d8b037e36be71b2f9a5afa9bb8cc695ec7a1e913"
           },
           {
             "@id": "composition/1/artifact/data_release_manifest",
             "@type": "trov:ResearchArtifact",
-            "schema:name": "policyengine-us-data release manifest 1.110.12",
+            "schema:name": "policyengine-us-data release manifest 1.113.1",
             "trov:mimeType": "application/json",
-            "trov:sha256": "17cfd2fbb31064834ed82c0fd7d8ae5c272fe7f24b1e48b226a4acf97ff4c5dd"
+            "trov:sha256": "d6b29ceff0cbf6a5cff4de94362ebc533dc5044c6a4155a46da7143140a8cb5f"
           },
           {
             "@id": "composition/1/artifact/dataset",
             "@type": "trov:ResearchArtifact",
             "schema:name": "enhanced_cps_2024",
             "trov:mimeType": "application/x-hdf5",
-            "trov:sha256": "58a6639f7511b8d804701417e2647f0c3a77f51a3d90441037eaf004b1f00761"
+            "trov:sha256": "0ce549af18753287c097718362b8cd5cdccfc47953acf7f282709d604cf314d2"
           },
           {
             "@id": "composition/1/artifact/model_wheel",
             "@type": "trov:ResearchArtifact",
-            "schema:name": "policyengine-us==1.690.7 wheel",
+            "schema:name": "policyengine-us==1.691.3 wheel",
             "trov:mimeType": "application/zip",
-            "trov:sha256": "5a7a541efabac98fa069d6845902cf5924c81db67383234b55dcd2b8bfcfc3ca"
+            "trov:sha256": "c5d37aa4442f23d48bd5d587a02876c89d83c6135809f12988cc39bd3a47e8b2"
           }
         ],
         "trov:hasFingerprint": {
           "@id": "composition/1/fingerprint",
           "@type": "trov:CompositionFingerprint",
-          "trov:sha256": "b84e895b3f19ffee5ec299b94ae2155448a12af2b3dc61d00f3d17003ecdf14a"
+          "trov:sha256": "316e373ed13360efa12037200719c9621ce7bced6d80acc4dfa7bbb72962892f"
         }
       },
       "trov:hasPerformance": {
         "@id": "trp/1",
         "@type": "trov:TransparentResearchPerformance",
-        "pe:builtWithModelVersion": "1.690.7",
+        "pe:builtWithModelVersion": "1.691.3",
         "pe:certifiedBy": "policyengine.py bundled manifest",
-        "pe:certifiedForModelVersion": "1.690.7",
-        "pe:ciGitRef": "refs/heads/main",
-        "pe:ciGitSha": "1718b493e4749faf62f0ffdff480205abdd20011",
-        "pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/25692889721",
+        "pe:certifiedForModelVersion": "1.691.3",
         "pe:compatibilityBasis": "exact_build_model_version",
-        "pe:dataBuildFingerprint": "sha256:9961ed1c5d00943a360724da560eee425eb9f99f91896f053dca74724c46e96e",
-        "pe:dataBuildId": "policyengine-us-data-1.110.12",
-        "pe:emittedIn": "github-actions",
-        "rdfs:comment": "Certification of build policyengine-us-data-1.110.12 for policyengine-us 1.690.7.",
+        "pe:dataBuildFingerprint": "sha256:d891044ece8ec3338904771c879b98ec11a12f1090c074e5b8cee846825d8056",
+        "pe:dataBuildId": "policyengine-us-data-1.113.1",
+        "pe:emittedIn": "local",
+        "rdfs:comment": "Certification of build policyengine-us-data-1.113.1 for policyengine-us 1.691.3.",
         "trov:accessedArrangement": {
           "@id": "arrangement/1"
         },
-        "trov:startedAtTime": "2026-05-11T18:53:05.508006Z",
+        "trov:startedAtTime": "2026-05-13T03:56:37.150215Z",
         "trov:wasConductedBy": {
           "@id": "trs"
         }

src/policyengine/provenance/bundle.py

@@ -395,6 +395,13 @@ def refresh_release_bundle(
     dataset_repo_id = data_artifact_json.get("repo_id", repo_id)
     dataset_path = data_artifact_json.get("path", dataset_path)
     dataset_revision = data_artifact_json.get("revision", new_data)
+    if (
+        release_manifest_json is not None
+        and new_release_manifest_revision is not None
+        and dataset_repo_id == repo_id
+        and dataset_revision == new_data
+    ):
+        dataset_revision = new_release_manifest_revision
 
     # Only hit HF if the data version actually changed.
     if new_data != old_data:

src/policyengine/provenance/manifest.py

@@ -1,3 +1,4 @@
+import hashlib
 import os
 from functools import lru_cache
 from importlib import import_module
@@ -127,6 +128,8 @@ class DataReleaseManifest(BaseModel):
     can enclose the full set of artifacts published together). Distinct
     from per-artifact DOIs on ``DataReleaseArtifact.preservation_mirrors``.
     Populated when the release pipeline mirrors to a DOI-minting host."""
+    source_sha256: Optional[str] = Field(default=None, exclude=True)
+    """Byte sha256 of the fetched manifest before runtime URI rewrites."""
 
 
 class DataCertification(BaseModel):
@@ -180,9 +183,13 @@ def https_dataset_uri(repo_id: str, path_in_repo: str, revision: str) -> str:
     return f"https://huggingface.co/{repo_id}/resolve/{revision}/{path_in_repo}"
 
 
+def _artifact_revision(data_package: "DataPackageVersion") -> str:
+    return data_package.release_manifest_revision or data_package.version
+
+
 def https_release_manifest_uri(data_package: "DataPackageVersion") -> str:
     """Return a dereferenceable HTTPS URI for a data release manifest."""
-    revision = data_package.release_manifest_revision or data_package.version
+    revision = _artifact_revision(data_package)
     return (
         f"https://huggingface.co/{data_package.repo_id}/resolve/"
         f"{revision}/{data_package.release_manifest_path}"
@@ -267,7 +274,20 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest:
         raise DataReleaseManifestUnavailableError(
             "Could not fetch the data release manifest from Hugging Face."
         ) from exc
-    return DataReleaseManifest.model_validate_json(response.text)
+    data_release_manifest = DataReleaseManifest.model_validate_json(response.text)
+    source_bytes = response.content
+    if not isinstance(source_bytes, bytes):
+        source_bytes = response.text.encode("utf-8")
+    data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest()
+    release_revision = country_manifest.data_package.release_manifest_revision
+    if release_revision is not None:
+        for artifact in data_release_manifest.artifacts.values():
+            if (
+                artifact.repo_id == country_manifest.data_package.repo_id
+                and artifact.revision == country_manifest.data_package.version
+            ):
+                artifact.revision = release_revision
+    return data_release_manifest
 
 
 def _specifier_matches(version: str, specifier: str) -> bool:
@@ -404,7 +424,7 @@ def resolve_dataset_reference(country_id: str, dataset: str) -> str:
         return build_hf_uri(
             repo_id=manifest.data_package.repo_id,
             path_in_repo=path_reference.path,
-            revision=manifest.data_package.version,
+            revision=_artifact_revision(manifest.data_package),
         )
 
     data_release_manifest = get_data_release_manifest(country_id)
@@ -525,5 +545,5 @@ def resolve_region_dataset_path(
     return build_hf_uri(
         repo_id=manifest.data_package.repo_id,
         path_in_repo=resolved_path,
-        revision=manifest.data_package.version,
+        revision=_artifact_revision(manifest.data_package),
     )

src/policyengine/provenance/trace.py

@@ -353,21 +353,14 @@ def build_trace_tro_from_release_bundle(
         if data_release_manifest is not None
         else None
     )
-    dataset_location = (
-        https_dataset_uri(
-            repo_id=dataset_artifact.repo_id,
-            path_in_repo=dataset_artifact.path,
-            revision=dataset_artifact.revision,
-        )
-        if dataset_artifact is not None
-        else _dataset_location_from_uri(certified_artifact.uri)
-    )
+    dataset_location = _dataset_location_from_uri(certified_artifact.uri)
 
     bundle_manifest_hash = hashlib.sha256(
         canonical_json_bytes(country_manifest.model_dump(mode="json"))
     ).hexdigest()
     data_release_manifest_hash = (
-        hashlib.sha256(
+        data_release_manifest.source_sha256
+        or hashlib.sha256(
             canonical_json_bytes(data_release_manifest.model_dump(mode="json"))
         ).hexdigest()
         if data_release_manifest is not None

tests/fixtures/household_calculator_snapshots/us_model_surface.json

@@ -5,7 +5,7 @@
   "has_income_tax": true,
   "has_region_registry": true,
   "model_package_name": "policyengine-us",
-  "num_parameters_bucketed_100s": 851,
+  "num_parameters_bucketed_100s": 852,
   "num_variables_bucketed_100s": 48,
   "region_registry_country": "us"
 }

tests/test_bundle_refresh.py

@@ -306,6 +306,10 @@ def fake_urlopen(request, *args, **kwargs):
         written["data_package"]["release_manifest_revision"]
         == "release-manifest-commit-sha"
     )
+    assert (
+        written["certified_data_artifact"]["uri"]
+        == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@release-manifest-commit-sha"
+    )
 
 
 def test__bump_data_only_falls_back_to_main_for_release_manifest(
@@ -337,6 +341,10 @@ def fake_urlopen(request, *args, **kwargs):
         written["data_package"]["release_manifest_revision"]
         == "release-manifest-commit-sha"
     )
+    assert (
+        written["certified_data_artifact"]["uri"]
+        == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@release-manifest-commit-sha"
+    )
 
 
 def test__release_manifest_version_mismatch_raises(sandbox) -> None:

tests/test_models.py

@@ -113,12 +113,12 @@ def test_has_release_manifest_metadata(self):
         assert us_latest.release_manifest is not None
         assert us_latest.release_manifest.country_id == "us"
         assert us_latest.model_package.name == "policyengine-us"
-        assert us_latest.model_package.version == "1.690.7"
+        assert us_latest.model_package.version == "1.691.3"
         assert us_latest.data_package.name == "policyengine-us-data"
-        assert us_latest.data_package.version == "1.110.12"
+        assert us_latest.data_package.version == "1.113.1"
         assert (
             us_latest.default_dataset_uri
-            == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.110.12"
+            == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@99e0ec7e784cdba43dd21ff1d80a081599a7a537"
         )
 
     def test_has_hundreds_of_parameters(self):