Skip to content
This repository was archived by the owner on Jul 2, 2026. It is now read-only.

Commit 2ca73b6

Browse files
anth-volkanth-volk
committed
Update package version
1 parent 4983d65 commit 2ca73b6

7 files changed

Lines changed: 341 additions & 17 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
## [1.114.0] - 2026-05-13
2+
3+
### Added
4+
5+
- Extract local H5 clone selection, entity reindexing, and variable cloning into dedicated build-output seams.
6+
7+
18
## [1.113.1] - 2026-05-13
29

310
### Fixed

‎changelog.d/965.added‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎docs/engineering/pipeline-map.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,14 @@ class AreaBuildRequest
878878

879879
Complete request for one local-area or national H5 file.
880880

881+
### `policyengine_us_data.build_outputs.selection.AreaSelector`
882+
883+
```python
884+
class AreaSelector
885+
```
886+
887+
Apply request geography filters to clone-level calibration weights.
888+
881889
### `policyengine_us_data.build_outputs.validation.AreaValidationService`
882890

883891
```python
@@ -894,6 +902,14 @@ class ArtifactIdentity
894902

895903
Stable identity for an input artifact used by traceability.
896904

905+
### `policyengine_us_data.build_outputs.selection.CloneSelection`
906+
907+
```python
908+
class CloneSelection
909+
```
910+
911+
Active clone rows selected for one H5 output.
912+
897913
### `policyengine_us_data.build_outputs.source_dataset.EntityGraph`
898914

899915
```python
@@ -902,6 +918,14 @@ class EntityGraph
902918

903919
Structural relationships between source dataset entities.
904920

921+
### `policyengine_us_data.build_outputs.reindexing.EntityReindexer`
922+
923+
```python
924+
class EntityReindexer
925+
```
926+
927+
Build sequential entity IDs and relationship arrays after clone selection.
928+
905929
### `policyengine_us_data.calibration.publish_local_area.compute_input_fingerprint`
906930

907931
```python
@@ -942,6 +966,14 @@ class PublishingInputBundle
942966

943967
Input artifact bundle for one local H5 publication scope.
944968

969+
### `policyengine_us_data.build_outputs.reindexing.ReindexedEntities`
970+
971+
```python
972+
class ReindexedEntities
973+
```
974+
975+
Entity IDs, relationship arrays, and source indices for one H5 output.
976+
945977
### `policyengine_us_data.build_outputs.geography_loader.ResolvedGeographySource`
946978

947979
```python
@@ -990,6 +1022,22 @@ class ValidationPolicy
9901022

9911023
Validation switch for a local H5 worker session.
9921024

1025+
### `policyengine_us_data.build_outputs.variables.VariableClonePayload`
1026+
1027+
```python
1028+
class VariableClonePayload
1029+
```
1030+
1031+
Cloned source variable arrays before H5-specific overrides.
1032+
1033+
### `policyengine_us_data.build_outputs.variables.VariableCloner`
1034+
1035+
```python
1036+
class VariableCloner
1037+
```
1038+
1039+
Clone source variable arrays using selected and reindexed entity rows.
1040+
9931041
### `policyengine_us_data.build_outputs.bootstrap.WorkerBootstrapBuilder`
9941042

9951043
```python

‎docs/generated/pipeline_api.json‎

Lines changed: 174 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -447,7 +447,7 @@
447447
"docstring": "Build city H5 files with checkpointing, optionally uploading.",
448448
"id": "build_cities",
449449
"kind": "function",
450-
"line": 947,
450+
"line": 802,
451451
"metadata": {
452452
"api_refs": [
453453
"policyengine_us_data.calibration.publish_local_area.build_cities"
@@ -506,7 +506,7 @@
506506
"docstring": "Build district H5 files with checkpointing, optionally uploading.",
507507
"id": "build_districts",
508508
"kind": "function",
509-
"line": 860,
509+
"line": 715,
510510
"metadata": {
511511
"api_refs": [
512512
"policyengine_us_data.calibration.publish_local_area.build_districts"
@@ -533,7 +533,7 @@
533533
"docstring": "Build an H5 file by cloning records for each nonzero weight.\n\nArgs:\n weights: Clone-level weight vector, shape (n_clones_total * n_hh,).\n geography: GeographyAssignment from assign_random_geography.\n dataset_path: Path to base dataset H5 file.\n output_path: Where to write the output H5 file.\n cd_subset: If provided, only include clones for these CDs.\n county_fips_filter: If provided, zero out weights for clones\n whose county FIPS is not in this set.\n takeup_filter: List of takeup vars to apply.\n\nReturns:\n Path to the output H5 file.",
534534
"id": "build_h5",
535535
"kind": "function",
536-
"line": 328,
536+
"line": 345,
537537
"metadata": {
538538
"api_refs": [
539539
"policyengine_us_data.calibration.publish_local_area.build_h5"
@@ -663,7 +663,7 @@
663663
"docstring": "Build state H5 files with checkpointing, optionally uploading.",
664664
"id": "build_states",
665665
"kind": "function",
666-
"line": 774,
666+
"line": 629,
667667
"metadata": {
668668
"api_refs": [
669669
"policyengine_us_data.calibration.publish_local_area.build_states"
@@ -1373,6 +1373,33 @@
13731373
"signature": "class AreaBuildRequest",
13741374
"source_file": "policyengine_us_data/build_outputs/requests.py"
13751375
},
1376+
"local_h5_area_selector": {
1377+
"docstring": "Apply request geography filters to clone-level calibration weights.",
1378+
"id": "local_h5_area_selector",
1379+
"kind": "class",
1380+
"line": 30,
1381+
"metadata": {
1382+
"api_refs": [
1383+
"policyengine_us_data.build_outputs.selection.AreaSelector"
1384+
],
1385+
"description": "Select active clone-household rows for one local H5 output.",
1386+
"id": "local_h5_area_selector",
1387+
"label": "AreaSelector",
1388+
"node_type": "library",
1389+
"pathways": [
1390+
"local_h5"
1391+
],
1392+
"source_file": "policyengine_us_data/build_outputs/selection.py",
1393+
"stability": "moving",
1394+
"status": "current",
1395+
"validation_commands": [
1396+
"uv run pytest tests/unit/build_outputs/test_selection.py"
1397+
]
1398+
},
1399+
"object_path": "policyengine_us_data.build_outputs.selection.AreaSelector",
1400+
"signature": "class AreaSelector",
1401+
"source_file": "policyengine_us_data/build_outputs/selection.py"
1402+
},
13761403
"local_h5_area_validation_service": {
13771404
"docstring": "Build validation state for all H5 requests handled by one worker.",
13781405
"id": "local_h5_area_validation_service",
@@ -1409,7 +1436,7 @@
14091436
"docstring": "Stable identity for an input artifact used by traceability.\n\nAttributes:\n logical_name: Semantic artifact name, such as `\"weights\"` or\n `\"source_dataset\"`.\n path: Physical artifact path when the artifact exists in the local\n runtime.\n sha256: Content digest prefixed with `\"sha256:\"`.\n size_bytes: Artifact size in bytes, when available.\n metadata: Additional normalized metadata, for example canonical\n geography checksum or source kind.",
14101437
"id": "local_h5_artifact_identity",
14111438
"kind": "class",
1412-
"line": 91,
1439+
"line": 92,
14131440
"metadata": {
14141441
"api_refs": [
14151442
"policyengine_us_data.build_outputs.fingerprinting.ArtifactIdentity"
@@ -1432,11 +1459,38 @@
14321459
"signature": "class ArtifactIdentity",
14331460
"source_file": "policyengine_us_data/build_outputs/fingerprinting.py"
14341461
},
1462+
"local_h5_clone_selection": {
1463+
"docstring": "Active clone rows selected for one H5 output.",
1464+
"id": "local_h5_clone_selection",
1465+
"kind": "class",
1466+
"line": 113,
1467+
"metadata": {
1468+
"api_refs": [
1469+
"policyengine_us_data.build_outputs.selection.CloneSelection"
1470+
],
1471+
"description": "Selected clone-household rows for one local H5 output.",
1472+
"id": "local_h5_clone_selection",
1473+
"label": "CloneSelection",
1474+
"node_type": "library",
1475+
"pathways": [
1476+
"local_h5"
1477+
],
1478+
"source_file": "policyengine_us_data/build_outputs/selection.py",
1479+
"stability": "moving",
1480+
"status": "current",
1481+
"validation_commands": [
1482+
"uv run pytest tests/unit/build_outputs/test_selection.py"
1483+
]
1484+
},
1485+
"object_path": "policyengine_us_data.build_outputs.selection.CloneSelection",
1486+
"signature": "class CloneSelection",
1487+
"source_file": "policyengine_us_data/build_outputs/selection.py"
1488+
},
14351489
"local_h5_entity_graph": {
14361490
"docstring": "Structural relationships between source dataset entities.\n\n`EntityGraph` is the canonical in-memory shape for the source entity spine.\nIt stores raw ID arrays plus derived household-membership maps so later H5\nbuilder seams can select households without rebuilding joins.\n\nAttributes:\n household_ids: Source household IDs, one per household row.\n person_household_ids: Household ID for each source person row.\n subentity_ids: Entity IDs by subentity key, such as `tax_unit`.\n person_subentity_ids: Entity ID for each source person row by subentity.\n household_to_person_indices: Household row index to person row indices.\n household_to_subentity_indices: Subentity key to household row index to\n subentity row indices.",
14371491
"id": "local_h5_entity_graph",
14381492
"kind": "class",
1439-
"line": 56,
1493+
"line": 77,
14401494
"metadata": {
14411495
"api_refs": [
14421496
"policyengine_us_data.build_outputs.source_dataset.EntityGraph"
@@ -1459,6 +1513,33 @@
14591513
"signature": "class EntityGraph",
14601514
"source_file": "policyengine_us_data/build_outputs/source_dataset.py"
14611515
},
1516+
"local_h5_entity_reindexer": {
1517+
"docstring": "Build sequential entity IDs and relationship arrays after clone selection.",
1518+
"id": "local_h5_entity_reindexer",
1519+
"kind": "class",
1520+
"line": 31,
1521+
"metadata": {
1522+
"api_refs": [
1523+
"policyengine_us_data.build_outputs.reindexing.EntityReindexer"
1524+
],
1525+
"description": "Reindex selected household, person, and subentity rows for local H5 outputs.",
1526+
"id": "local_h5_entity_reindexer",
1527+
"label": "EntityReindexer",
1528+
"node_type": "library",
1529+
"pathways": [
1530+
"local_h5"
1531+
],
1532+
"source_file": "policyengine_us_data/build_outputs/reindexing.py",
1533+
"stability": "moving",
1534+
"status": "current",
1535+
"validation_commands": [
1536+
"uv run pytest tests/unit/build_outputs/test_reindexing.py"
1537+
]
1538+
},
1539+
"object_path": "policyengine_us_data.build_outputs.reindexing.EntityReindexer",
1540+
"signature": "class EntityReindexer",
1541+
"source_file": "policyengine_us_data/build_outputs/reindexing.py"
1542+
},
14621543
"local_h5_input_fingerprint": {
14631544
"docstring": "",
14641545
"id": "local_h5_input_fingerprint",
@@ -1491,7 +1572,7 @@
14911572
"docstring": "Lazy holder-backed variable reader for a source microsimulation.\n\nThe provider intentionally reads arrays only when callers request a\nvariable/period pair. It caches the normalized array for repeated access\nwhile keeping construction lightweight.\n\nAttributes:\n simulation: Source `policyengine_us.Microsimulation` or a compatible\n test double with `input_variables` and `get_holder(...)`.",
14921573
"id": "local_h5_microsimulation_variable_provider",
14931574
"kind": "class",
1494-
"line": 377,
1575+
"line": 384,
14951576
"metadata": {
14961577
"api_refs": [
14971578
"policyengine_us_data.build_outputs.source_dataset.MicrosimulationVariableProvider"
@@ -1545,7 +1626,7 @@
15451626
"docstring": "Read PolicyEngine source H5 files into `SourceDatasetSnapshot` objects.",
15461627
"id": "local_h5_policyengine_dataset_reader",
15471628
"kind": "class",
1548-
"line": 541,
1629+
"line": 560,
15491630
"metadata": {
15501631
"api_refs": [
15511632
"policyengine_us_data.build_outputs.source_dataset.PolicyEngineDatasetReader"
@@ -1572,7 +1653,7 @@
15721653
"docstring": "Input artifact bundle for one local H5 publication scope.\n\nThe bundle is the library-level contract used by coordinators before\nfingerprinting. Paths point to local files already materialized in the\nworker or orchestration environment.\n\nAttributes:\n weights_path: Path to `calibration_weights.npy`.\n source_dataset_path: Path to the source-imputed dataset H5.\n target_db_path: Optional `policy_data.db` path used for validation.\n exact_geography_path: Optional saved `geography_assignment.npz` path.\n calibration_package_path: Optional `calibration_package.pkl` path used\n as a geography fallback.\n run_config_path: Optional run configuration JSON with code and model\n build metadata.\n run_id: Pipeline run identifier.\n version: Package or release version associated with the publish.\n n_clones: Expected clone count, when known.\n seed: Geography assignment seed used by the build.\n legacy_blocks_path: Optional legacy `stacked_blocks.npy` fallback.",
15731654
"id": "local_h5_publishing_input_bundle",
15741655
"kind": "class",
1575-
"line": 41,
1656+
"line": 42,
15761657
"metadata": {
15771658
"api_refs": [
15781659
"policyengine_us_data.build_outputs.fingerprinting.PublishingInputBundle"
@@ -1592,6 +1673,33 @@
15921673
"signature": "class PublishingInputBundle",
15931674
"source_file": "policyengine_us_data/build_outputs/fingerprinting.py"
15941675
},
1676+
"local_h5_reindexed_entities": {
1677+
"docstring": "Entity IDs, relationship arrays, and source indices for one H5 output.",
1678+
"id": "local_h5_reindexed_entities",
1679+
"kind": "class",
1680+
"line": 123,
1681+
"metadata": {
1682+
"api_refs": [
1683+
"policyengine_us_data.build_outputs.reindexing.ReindexedEntities"
1684+
],
1685+
"description": "Output entity IDs and source row indices after local H5 clone selection.",
1686+
"id": "local_h5_reindexed_entities",
1687+
"label": "ReindexedEntities",
1688+
"node_type": "library",
1689+
"pathways": [
1690+
"local_h5"
1691+
],
1692+
"source_file": "policyengine_us_data/build_outputs/reindexing.py",
1693+
"stability": "moving",
1694+
"status": "current",
1695+
"validation_commands": [
1696+
"uv run pytest tests/unit/build_outputs/test_reindexing.py"
1697+
]
1698+
},
1699+
"object_path": "policyengine_us_data.build_outputs.reindexing.ReindexedEntities",
1700+
"signature": "class ReindexedEntities",
1701+
"source_file": "policyengine_us_data/build_outputs/reindexing.py"
1702+
},
15951703
"local_h5_resolved_geography_source": {
15961704
"docstring": "Resolved physical source used to recover calibration geography.\n\nAttributes:\n kind: Source format selected by `CalibrationGeographyLoader`.\n path: Path to the selected source artifact.",
15971705
"id": "local_h5_resolved_geography_source",
@@ -1623,7 +1731,7 @@
16231731
"docstring": "Explicit in-memory worker view of a source H5 dataset.\n\nThe snapshot groups source dataset structure and lazy variable access. It\ndoes not serialize artifacts or change current worker execution paths.\n\nAttributes:\n dataset_path: Source H5 dataset path.\n time_period: Default source calculation period.\n entity_graph: Source entity relationship graph.\n input_variables: Input variable names available from the source.\n variable_provider: Lazy variable reader for the source simulation.",
16241732
"id": "local_h5_source_dataset_snapshot",
16251733
"kind": "class",
1626-
"line": 464,
1734+
"line": 483,
16271735
"metadata": {
16281736
"api_refs": [
16291737
"policyengine_us_data.build_outputs.source_dataset.SourceDatasetSnapshot"
@@ -1650,7 +1758,7 @@
16501758
"docstring": "Build traceability bundles and derive deterministic scope fingerprints.\n\nThe service centralizes provenance rules for local H5 publishing. It avoids\nimporting heavy simulation machinery until a fallback record-count inference\nis needed.",
16511759
"id": "local_h5_traceability",
16521760
"kind": "class",
1653-
"line": 200,
1761+
"line": 201,
16541762
"metadata": {
16551763
"api_refs": [
16561764
"policyengine_us_data.build_outputs.fingerprinting.FingerprintingService"
@@ -1677,7 +1785,7 @@
16771785
"docstring": "Full provenance record for one local H5 publish scope.\n\nAttributes:\n scope: Publish scope being fingerprinted, currently `\"regional\"` or\n `\"national\"`.\n weights: Identity of the calibration weights artifact.\n source_dataset: Identity of the source-imputed H5 artifact.\n exact_geography: Identity of the geography source used for clone\n selection, if available.\n target_db: Optional target database identity.\n calibration_package: Optional calibration package identity.\n run_config: Optional run configuration identity.\n code_version: Normalized code version metadata extracted from run\n config.\n model_build: Normalized model build metadata extracted from run config.\n metadata: Scope-level metadata such as run ID, version, clone count, and\n seed.",
16781786
"id": "local_h5_traceability_bundle",
16791787
"kind": "class",
1680-
"line": 126,
1788+
"line": 127,
16811789
"metadata": {
16821790
"api_refs": [
16831791
"policyengine_us_data.build_outputs.fingerprinting.TraceabilityBundle"
@@ -1754,6 +1862,60 @@
17541862
"signature": "class ValidationPolicy",
17551863
"source_file": "policyengine_us_data/build_outputs/validation.py"
17561864
},
1865+
"local_h5_variable_clone_payload": {
1866+
"docstring": "Cloned source variable arrays before H5-specific overrides.",
1867+
"id": "local_h5_variable_clone_payload",
1868+
"kind": "class",
1869+
"line": 138,
1870+
"metadata": {
1871+
"api_refs": [
1872+
"policyengine_us_data.build_outputs.variables.VariableClonePayload"
1873+
],
1874+
"description": "Period-grouped cloned source variables for one local H5 output.",
1875+
"id": "local_h5_variable_clone_payload",
1876+
"label": "VariableClonePayload",
1877+
"node_type": "library",
1878+
"pathways": [
1879+
"local_h5"
1880+
],
1881+
"source_file": "policyengine_us_data/build_outputs/variables.py",
1882+
"stability": "moving",
1883+
"status": "current",
1884+
"validation_commands": [
1885+
"uv run pytest tests/unit/build_outputs/test_variables.py"
1886+
]
1887+
},
1888+
"object_path": "policyengine_us_data.build_outputs.variables.VariableClonePayload",
1889+
"signature": "class VariableClonePayload",
1890+
"source_file": "policyengine_us_data/build_outputs/variables.py"
1891+
},
1892+
"local_h5_variable_cloner": {
1893+
"docstring": "Clone source variable arrays using selected and reindexed entity rows.\n\nThe skip/error policy intentionally mirrors the first local H5 pipeline\nimplementation. Variables outside the local-H5 allowlist are skipped before\nholder access. Allowed variables are skipped when they have no known periods\nor belong to an entity that local H5s do not clone. Malformed metadata for\nan allowed variable is not skipped; it should fail so incomplete outputs are\nnot produced silently.",
1894+
"id": "local_h5_variable_cloner",
1895+
"kind": "class",
1896+
"line": 49,
1897+
"metadata": {
1898+
"api_refs": [
1899+
"policyengine_us_data.build_outputs.variables.VariableCloner"
1900+
],
1901+
"description": "Clone selected source variables into period-grouped local H5 payloads.",
1902+
"id": "local_h5_variable_cloner",
1903+
"label": "VariableCloner",
1904+
"node_type": "library",
1905+
"pathways": [
1906+
"local_h5"
1907+
],
1908+
"source_file": "policyengine_us_data/build_outputs/variables.py",
1909+
"stability": "moving",
1910+
"status": "current",
1911+
"validation_commands": [
1912+
"uv run pytest tests/unit/build_outputs/test_variables.py"
1913+
]
1914+
},
1915+
"object_path": "policyengine_us_data.build_outputs.variables.VariableCloner",
1916+
"signature": "class VariableCloner",
1917+
"source_file": "policyengine_us_data/build_outputs/variables.py"
1918+
},
17571919
"local_h5_worker_bootstrap_builder": {
17581920
"docstring": "Build and persist one scope's local H5 worker bootstrap artifacts.",
17591921
"id": "local_h5_worker_bootstrap_builder",

0 commit comments

Comments
 (0)