pipeline_map.yaml

version: 1
canonical_stages:
- id: 1_build_datasets
  label: Stage 1
  title: Build Datasets
  description: Produce raw, base, extended, enhanced, stratified, source-imputed, and staged base datasets.
  manifest_step_ids:
  - 01_build_datasets
  - 04_stage_base_datasets
- id: 2_build_calibration_package
  label: Stage 2
  title: Build Calibration Package
  description: Build the calibration target package, geography tables, constraints, sparse matrices, and supporting metadata.
  manifest_step_ids:
  - 02_build_package
- id: 3_fit_weights
  label: Stage 3
  title: Fit Weights
  description: Fit calibration weights for regional and national output builds.
  manifest_step_ids:
  - 03_fit_weights_regional
  - 03_fit_weights_national
- id: 4_build_outputs
  label: Stage 4
  title: Build Outputs
  description: Build local-area and national H5 outputs and upload diagnostics.
  manifest_step_ids:
  - 04_build_h5_regional
  - 04_build_h5_national
  - 04_upload_diagnostics
- id: 5_validate_and_promote_release
  label: Stage 5
  title: Validate and Promote Release
  description: Validate staged artifacts, promote release outputs, and finalize publication manifests.
  manifest_step_ids:
  - 05_promote_release
stages:
- id: 1a_raw_data_download
  label: 1a
  title: 'Substage 1a: Raw Data Download'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: '0'
  manifest_step_ids:
  - 01_build_datasets
  description: Download raw survey data from Census, IRS, Federal Reserve, and HuggingFace
  country: us
  extra_nodes:
  - id: cps_url
    label: Census CPS ASEC
    node_type: artifact
    description: ZIP with CSV files from www2.census.gov
  - id: acs_url
    label: Census ACS PUMS
    node_type: artifact
    description: Person + Household CSVs, 1-Year estimates
  - id: scf_url
    label: Federal Reserve SCF
    node_type: artifact
    description: Stata format (.dta), Survey of Consumer Finances
  - id: hf_private
    label: HuggingFace Private Repo
    node_type: external
    description: policyengine/irs-soi-puf - requires HUGGING_FACE_TOKEN
  - id: hf_public
    label: HuggingFace Public Repo
    node_type: external
    description: policyengine/policyengine-us-data - SIPP, block distributions, policy_data.db
  - id: download_http
    label: HTTP Download + ZIP Extract
    node_type: process
    description: requests.get() with streaming, tqdm progress bar
  - id: download_hf
    label: HuggingFace Hub Download
    node_type: process
    description: hf_hub_download(), token-authenticated for private repos
  - id: csv_parse
    label: CSV/Stata Parsing
    node_type: process
    description: pandas read_csv/read_stata, entity table construction
  - id: out_cps_raw
    label: census_cps_2024.h5
    node_type: artifact
    description: '5 entity tables: Person, Family, Household, Tax Unit, SPM Unit'
  - id: out_acs_raw
    label: census_acs_2022.h5
    node_type: artifact
    description: Person + Household tables
  - id: out_puf_raw
    label: irs_puf_2015.h5
    node_type: artifact
    description: PUF + Demographics tables
  - id: out_soi
    label: soi.csv
    node_type: artifact
    description: SOI aggregate statistics
  - id: out_scf
    label: SCF raw data
    node_type: artifact
    description: Stata -> DataFrame, auto loans + net worth
  - id: out_sipp
    label: pu2023_slim.csv
    node_type: artifact
    description: SIPP microdata (pipe-delimited)
  - id: out_block
    label: block_cd_distributions.csv.gz
    node_type: artifact
    description: Census block populations, block-to-CD crosswalk
  - id: out_pop
    label: np2023_d5_mid.csv
    node_type: artifact
    description: Census population projections
  - id: out_calibration_db
    label: policy_data.db
    node_type: artifact
    description: SQLite calibration targets database
  - id: util_storage
    label: STORAGE_FOLDER
    node_type: utility
    description: policyengine_us_data/storage/ - all downloads cached here
  edges:
  - source: cps_url
    target: download_http
    edge_type: external_source
    label: CPS ASEC ZIP
  - source: acs_url
    target: download_http
    edge_type: external_source
    label: ACS PUMS CSV
  - source: scf_url
    target: download_http
    edge_type: external_source
    label: SCF .dta
  - source: hf_private
    target: download_hf
    edge_type: external_source
    label: PUF, demographics, SOI, pop
  - source: hf_public
    target: download_hf
    edge_type: external_source
    label: SIPP, block, policy_data.db
  - source: download_http
    target: csv_parse
    edge_type: data_flow
    label: raw files
  - source: download_hf
    target: csv_parse
    edge_type: data_flow
    label: raw files
  - source: csv_parse
    target: out_cps_raw
    edge_type: produces_artifact
    label: census_cps_2024.h5
  - source: csv_parse
    target: out_acs_raw
    edge_type: produces_artifact
    label: census_acs_2022.h5
  - source: csv_parse
    target: out_puf_raw
    edge_type: produces_artifact
    label: irs_puf_2015.h5
  - source: csv_parse
    target: out_soi
    edge_type: produces_artifact
    label: soi.csv
  - source: download_http
    target: out_scf
    edge_type: produces_artifact
    label: SCF raw data
  - source: download_hf
    target: out_sipp
    edge_type: produces_artifact
    label: pu2023_slim.csv
  - source: download_hf
    target: out_block
    edge_type: produces_artifact
    label: block_cd_distributions.csv.gz
  - source: download_hf
    target: out_pop
    edge_type: produces_artifact
    label: np2023_d5_mid.csv
  - source: download_hf
    target: out_calibration_db
    edge_type: produces_artifact
    label: policy_data.db
  status: current
  stability: moving
- id: 1b_base_dataset_construction
  label: 1b
  title: 'Substage 1b: Base Dataset Construction'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: '1'
  manifest_step_ids:
  - 01_build_datasets
  description: Build CPS 2024 and PUF 2024 from raw survey data, donor-based labor-market imputations, and retirement contribution inference
  country: us
  extra_nodes:
  - id: in_census_cps
    label: census_cps_2024.h5
    node_type: artifact
    description: Raw CPS ASEC from substage 1a
  - id: in_census_cps_prev
    label: census_cps_2023.h5
    node_type: artifact
    description: Previous year CPS for income matching
  - id: in_acs
    label: ACS 2022
    node_type: artifact
    description: Training data for rent QRF
  - id: in_sipp
    label: SIPP 2023
    node_type: artifact
    description: Training data for tips QRF
  - id: in_scf
    label: SCF 2022
    node_type: artifact
    description: Training data for auto loans QRF
  - id: in_org
    label: CPS Basic ORG 2024
    node_type: external
    description: Monthly CPS basic ORG wage microdata used to train labor-market imputations
  - id: in_uprating
    label: uprating_factors.csv
    node_type: artifact
    description: PE uprating factors table
  - id: out_cps
    label: cps_2024.h5
    node_type: artifact
    description: Dataset.ARRAYS format, ~65K households (half-sample)
  - id: out_puf
    label: puf_2024.h5
    node_type: artifact
    description: Dataset.ARRAYS format
  - id: in_irs_puf
    label: irs_puf_2015.h5
    node_type: artifact
    description: Raw IRS PUF from substage 1a
  - id: in_demographics
    label: demographics_2015.csv
    node_type: artifact
    description: PUF demographics from HuggingFace
  - id: in_cps_pension
    label: CPS_2024 / CPS_2021
    node_type: artifact
    description: CPS donor sample for pension QRF; falls back to CPS_2021 if current CPS is unavailable
  - id: util_seeded_rng
    label: seeded_rng()
    node_type: utility
    description: Deterministic per-variable RNG
  - id: util_qrf
    label: microimpute QRF
    node_type: utility
    description: fit_predict() for sequential imputation
  - id: util_retirement_limits
    label: get_retirement_limits()
    node_type: utility
    description: IRS contribution limits
  edges:
  - source: in_census_cps
    target: add_id_variables
    edge_type: data_flow
    label: raw CPS tables
  - source: add_id_variables
    target: add_personal_variables
    edge_type: data_flow
  - source: add_personal_variables
    target: add_personal_income_variables
    edge_type: data_flow
  - source: add_personal_income_variables
    target: add_previous_year_income
    edge_type: data_flow
  - source: in_census_cps_prev
    target: add_previous_year_income
    edge_type: data_flow
    label: prior year PERIDNUM
  - source: add_previous_year_income
    target: add_ssn_card_type
    edge_type: data_flow
  - source: add_ssn_card_type
    target: add_spm_variables
    edge_type: data_flow
  - source: add_spm_variables
    target: add_household_variables
    edge_type: data_flow
  - source: add_household_variables
    target: add_rent
    edge_type: data_flow
  - source: in_acs
    target: add_rent
    edge_type: external_source
    label: ACS training data
  - source: add_rent
    target: add_tips
    edge_type: data_flow
  - source: in_sipp
    target: add_tips
    edge_type: external_source
    label: SIPP training data
  - source: add_tips
    target: add_org_inputs
    edge_type: data_flow
  - source: in_org
    target: add_org_inputs
    edge_type: external_source
    label: ORG donor data
  - source: add_org_inputs
    target: add_auto_loan
    edge_type: data_flow
  - source: in_scf
    target: add_auto_loan
    edge_type: external_source
    label: SCF training data
  - source: add_auto_loan
    target: add_takeup
    edge_type: data_flow
  - source: add_takeup
    target: downsample
    edge_type: data_flow
  - source: downsample
    target: out_cps
    edge_type: produces_artifact
    label: cps_2024.h5
  - source: in_irs_puf
    target: preprocess_puf
    edge_type: data_flow
    label: raw PUF records
  - source: preprocess_puf
    target: simulate_qbi
    edge_type: data_flow
  - source: simulate_qbi
    target: impute_puf_demographics
    edge_type: data_flow
  - source: in_demographics
    target: impute_puf_demographics
    edge_type: data_flow
    label: demographics_2015.csv
  - source: impute_puf_demographics
    target: impute_puf_pension
    edge_type: data_flow
  - source: in_cps_pension
    target: impute_puf_pension
    edge_type: data_flow
    label: CPS donor sample
  - source: impute_puf_pension
    target: mortgage_convert
    edge_type: data_flow
  - source: mortgage_convert
    target: out_puf
    edge_type: produces_artifact
    label: puf_2024.h5
  - source: in_uprating
    target: out_puf
    edge_type: data_flow
    label: SOI growth rates
  - source: util_seeded_rng
    target: add_takeup
    edge_type: uses_utility
  - source: util_qrf
    target: add_rent
    edge_type: uses_utility
  - source: util_qrf
    target: add_tips
    edge_type: uses_utility
  - source: util_qrf
    target: add_org_inputs
    edge_type: uses_utility
  - source: util_qrf
    target: add_auto_loan
    edge_type: uses_utility
  - source: util_retirement_limits
    target: add_personal_income_variables
    edge_type: uses_utility
  - source: util_qrf
    target: impute_puf_demographics
    edge_type: uses_utility
  - source: util_qrf
    target: impute_puf_pension
    edge_type: uses_utility
  status: current
  stability: moving
- id: 1c_extended_cps_puf_clone
  label: 1c
  title: 'Substage 1c: Extended CPS (PUF Clone)'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: '2'
  manifest_step_ids:
  - 01_build_datasets
  description: Merge CPS + PUF via cloning, rematch clone features, QRF-impute incomes and CPS-only vars, then finalize Extended CPS inputs
  country: us
  extra_nodes:
  - id: in_cps_s2
    label: CPS_2024_Full
    node_type: artifact
    description: From Stage 1 (full sample)
  - id: in_puf_s2
    label: PUF_2024
    node_type: artifact
    description: From Stage 1
  - id: in_blocks_s2
    label: block_cd_distributions.csv.gz
    node_type: artifact
    description: Census block populations
  - id: in_scf_s2
    label: SCF_2022
    node_type: artifact
    description: From substage 1a (mortgage-balance donor sample)
  - id: geo_assign_s2
    label: Geography Assignment
    node_type: process
    description: assign_random_geography() - population-weighted block draw
  - id: out_ext
    label: extended_cps_2024.h5
    node_type: artifact
    description: ~260K households (doubled), CPS half + PUF half
  - id: util_qrf_s2
    label: microimpute QRF
    node_type: utility
    description: fit_predict() for sequential imputation
  - id: util_knn_s2
    label: sklearn NearestNeighbors
    node_type: utility
    description: Role-aware donor matching on standardized clone predictors
  edges:
  - source: in_cps_s2
    target: geo_assign_s2
    edge_type: data_flow
    label: CPS records
  - source: in_blocks_s2
    target: geo_assign_s2
    edge_type: data_flow
    label: block populations
  - source: in_puf_s2
    target: record_double
    edge_type: data_flow
    label: PUF records
  - source: in_cps_s2
    target: record_double
    edge_type: data_flow
    label: CPS records
  - source: geo_assign_s2
    target: record_double
    edge_type: data_flow
  - source: record_double
    target: puf_qrf_pass
    edge_type: data_flow
  - source: puf_qrf_pass
    target: retire_impute
    edge_type: data_flow
  - source: puf_qrf_pass
    target: weeks_impute
    edge_type: data_flow
  - source: retire_impute
    target: ss_reconcile
    edge_type: data_flow
  - source: weeks_impute
    target: ss_reconcile
    edge_type: data_flow
  - source: ss_reconcile
    target: clone_features
    edge_type: data_flow
  - source: clone_features
    target: cps_only
    edge_type: data_flow
  - source: cps_only
    target: qrf_pass2
    edge_type: data_flow
  - source: qrf_pass2
    target: mortgage_hints
    edge_type: data_flow
  - source: in_scf_s2
    target: mortgage_hints
    edge_type: data_flow
    label: SCF donor sample
  - source: mortgage_hints
    target: mortgage_convert
    edge_type: data_flow
  - source: mortgage_convert
    target: computed_export_contract
    edge_type: data_flow
  - source: computed_export_contract
    target: out_ext
    edge_type: produces_artifact
  - source: util_qrf_s2
    target: puf_qrf_pass
    edge_type: uses_utility
  - source: util_qrf_s2
    target: cps_only
    edge_type: uses_utility
  - source: util_qrf_s2
    target: mortgage_hints
    edge_type: uses_utility
  - source: util_knn_s2
    target: clone_features
    edge_type: uses_utility
  status: current
  stability: moving
- id: 1d_enhanced_cps_reweighting
  label: 1d
  title: 'Substage 1d: Enhanced CPS Reweighting'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: 3a
  manifest_step_ids:
  - 01_build_datasets
  description: Reweight Extended CPS to match national IRS/Census/CBO targets, then apply the 2025 ACA post-calibration override (deprecated ECPS pathway)
  country: us
  extra_nodes:
  - id: in_ext_half
    label: ExtendedCPS_2024_Half
    node_type: artifact
    description: Half-sample from Stage 2
  - id: build_loss
    label: build_loss_matrix()
    node_type: process
    description: Hundreds of calibration targets - returns (matrix, target_vector)
  - id: t_soi
    label: IRS SOI
    node_type: external
    description: AGI, income types, filer counts by AGI band / filing status
  - id: t_census
    label: Census Population
    node_type: external
    description: 86 single-year age groups from np2023_d5_mid.csv
  - id: t_cbo
    label: CBO Budget
    node_type: external
    description: Income tax, SNAP, SS, SSI, UC
  - id: t_state
    label: State Targets
    node_type: process
    description: 50 states + DC - population, AGI, ACA, SNAP, age groups
  - id: t_jct
    label: JCT Tax Expenditures
    node_type: external
    description: SALT, medical, charitable - counterfactual simulations
  - id: weight_validate
    label: Weight Validation
    node_type: process
    description: No NaN, no negatives - 100M < total HH < 200M
  - id: out_enhanced
    label: enhanced_cps_2024.h5
    node_type: artifact
    description: Reweighted production simulation dataset
  - id: util_loss
    label: build_loss_matrix()
    node_type: utility
    description: utils/loss.py
  - id: util_l0_s3
    label: HardConcrete L0
    node_type: utility
    description: utils/l0.py
  edges:
  - source: in_ext_half
    target: build_loss
    edge_type: data_flow
  - source: t_soi
    target: build_loss
    edge_type: external_source
  - source: t_census
    target: build_loss
    edge_type: external_source
  - source: t_cbo
    target: build_loss
    edge_type: external_source
  - source: t_jct
    target: build_loss
    edge_type: external_source
  - source: t_state
    target: build_loss
    edge_type: external_source
  - source: build_loss
    target: reweight
    edge_type: data_flow
    label: (matrix, targets)
  - source: reweight
    target: weight_validate
    edge_type: data_flow
  - source: weight_validate
    target: aca_2025_override
    edge_type: data_flow
  - source: aca_2025_override
    target: out_enhanced
    edge_type: produces_artifact
  - source: util_loss
    target: build_loss
    edge_type: uses_utility
  - source: util_l0_s3
    target: reweight
    edge_type: uses_utility
  status: current
  stability: moving
- id: 1e_stratified_cps
  label: 1e
  title: 'Substage 1e: Stratified CPS'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: 3b
  manifest_step_ids:
  - 01_build_datasets
  description: Stratify Extended CPS by income - keep top 1%, sample remaining 99%
  country: us
  groups:
  - id: create_stratified
    label: create_stratified_cps_dataset()
    description: Wrapper around AGI calculation, top-1% retention, and sampling into the stratified CPS artifact
    node_ids:
    - calc_agi
    - strat_top
    - strat_sample
    - out_strat
  extra_nodes:
  - id: in_ext_cps
    label: extended_cps_2024.h5
    node_type: artifact
    description: From Stage 2 (~260K HH)
  - id: calc_agi
    label: Calculate AGI
    node_type: process
    description: Microsimulation.calculate() - adjusted_gross_income mapped to household
  - id: strat_top
    label: Retain Top 1% by AGI
    node_type: process
    description: All high-income households kept - preserves tail representation
  - id: strat_sample
    label: Uniform Sample Remaining 99%
    node_type: process
    description: 'Target: ~12,000 households - optional 1.5x oversample of bottom 25%'
  - id: out_strat
    label: stratified_extended_cps_2024.h5
    node_type: artifact
    description: ~12K households - input to Stages 4-8
  edges:
  - source: in_ext_cps
    target: calc_agi
    edge_type: data_flow
  - source: calc_agi
    target: strat_top
    edge_type: data_flow
  - source: strat_top
    target: strat_sample
    edge_type: data_flow
  - source: strat_top
    target: out_strat
    edge_type: data_flow
    label: top 1%
  - source: strat_sample
    target: out_strat
    edge_type: data_flow
    label: sampled 99%
  status: current
  stability: moving
- id: 1f_source_imputation
  label: 1f
  title: 'Substage 1f: Source Imputation (ACS + SIPP + SCF)'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: '4'
  manifest_step_ids:
  - 01_build_datasets
  description: Impute wealth/assets from external surveys onto stratified CPS via QRF
  country: us
  extra_nodes:
  - id: in_strat_s4
    label: stratified_extended_cps_2024.h5
    node_type: artifact
    description: From Stage 3 (~12K HH)
  - id: in_acs_s4
    label: ACS_2022
    node_type: artifact
    description: American Community Survey - has state_fips predictor
  - id: in_sipp_s4
    label: SIPP 2023
    node_type: external
    description: pu2023_slim.csv from HuggingFace
  - id: in_scf_s4
    label: SCF_2022
    node_type: artifact
    description: Survey of Consumer Finances - 50% random subsample
  - id: sipp_assets_qrf
    label: SIPP Assets QRF
    node_type: process
    description: 5 predictors - imputes bank_account_assets, stock_assets, bond_assets
  - id: out_imputed
    label: source_imputed_stratified_extended_cps.h5
    node_type: artifact
    description: Enriched with ACS/SIPP/SCF vars - uploaded to HuggingFace
  - id: util_clone_assign
    label: clone_and_assign.py
    node_type: utility
    description: Geography assignment
  - id: util_qrf_s4
    label: microimpute QRF
    node_type: utility
    description: fit_predict() for sequential imputation
  edges:
  - source: in_strat_s4
    target: geo_assign
    edge_type: data_flow
  - source: geo_assign
    target: acs_qrf
    edge_type: data_flow
    label: state_fips
  - source: in_acs_s4
    target: acs_qrf
    edge_type: data_flow
  - source: in_sipp_s4
    target: sipp_qrf
    edge_type: external_source
  - source: in_sipp_s4
    target: sipp_assets_qrf
    edge_type: external_source
  - source: in_scf_s4
    target: scf_qrf
    edge_type: external_source
  - source: acs_qrf
    target: sipp_qrf
    edge_type: data_flow
    label: chain
  - source: sipp_qrf
    target: sipp_assets_qrf
    edge_type: data_flow
    label: chain
  - source: sipp_assets_qrf
    target: scf_qrf
    edge_type: data_flow
    label: chain
  - source: scf_qrf
    target: out_imputed
    edge_type: produces_artifact
  - source: util_clone_assign
    target: geo_assign
    edge_type: uses_utility
  - source: util_qrf_s4
    target: acs_qrf
    edge_type: uses_utility
  - source: util_qrf_s4
    target: sipp_qrf
    edge_type: uses_utility
  - source: util_qrf_s4
    target: sipp_assets_qrf
    edge_type: uses_utility
  - source: util_qrf_s4
    target: scf_qrf
    edge_type: uses_utility
  status: current
  stability: moving
- id: 1g_stage_base_datasets
  label: 1g
  title: 'Substage 1g: Stage Base Datasets'
  canonical_stage_id: 1_build_datasets
  legacy_stage_id: '7'
  manifest_step_ids:
  - 04_stage_base_datasets
  description: Stage base source-imputed datasets and policy database artifacts for the run
  country: us
  extra_nodes:
  - id: in_source_imputed_s1g
    label: source_imputed_*.h5
    node_type: artifact
    description: Base dataset artifacts copied into the pipeline volume
  - id: in_policy_db_s1g
    label: policy_data.db
    node_type: artifact
    description: Policy target database copied into the pipeline volume
  - id: hf_staging_base_s1g
    label: HuggingFace staging/{candidate_version}/{run_id}
    node_type: external
    description: Run-scoped staging prefix for base datasets
  - id: stage_base_datasets
    label: stage base datasets
    node_type: process
    description: Record base source-imputed datasets and policy database artifacts as staged run outputs.
    source_file: modal_app/pipeline.py
    status: current
    stability: moving
    validation_commands:
    - uv run pytest tests/unit/test_pipeline.py
  - id: out_staged_base_s1g
    label: staged base datasets
    node_type: artifact
    description: Base artifacts uploaded to staging paths for later promotion
  edges:
  - source: in_source_imputed_s1g
    target: stage_base_datasets
    edge_type: data_flow
  - source: in_policy_db_s1g
    target: stage_base_datasets
    edge_type: data_flow
  - source: stage_base_datasets
    target: out_staged_base_s1g
    edge_type: produces_artifact
  - source: out_staged_base_s1g
    target: hf_staging_base_s1g
    edge_type: data_flow
    label: uploaded to
  status: current
  stability: moving
- id: 2a_matrix_build_calibration_target_construction
  label: 2a
  title: 'Substage 2a: Matrix Build (Calibration Target Construction)'
  canonical_stage_id: 2_build_calibration_package
  legacy_stage_id: '5'
  manifest_step_ids:
  - 02_build_package
  description: Build sparse calibration matrix (targets x households x clones)
  country: us
  groups:
  - id: run_calibration_build
    label: run_calibration()
    description: 'Build phase: resolve targets and constraints, assemble clone values, and package the sparse calibration matrix'
    node_ids:
    - stage2_input_bundle
    - stage2_build_context
    - stage2_target_config_identity
    - stage2_target_catalog_load
    - stage2_target_catalog_reader
    - stage2_target_selection_policy
    - stage2_target_selection_result
    - stage2_geography_assignment_spec
    - stage2_geography_assignment_result
    - target_resolve
    - stage2_target_config_apply
    - target_uprate
    - geo_build
    - constraint_resolve
    - state_precomp
    - clone_assembly
    - takeup_rerand
    - sparse_build
    - build_matrix
    - build_matrix_chunked
    - stage2_artifact_specs
    - stage2_payload_boundary
    - stage2_payload_writer
    - stage2_payload_reader
    - stage2_calibration_package_writer
    - out_pkg
    - out_metadata
    - out_targets
    - out_target_facets
    - out_geography_summary
    - stage2_calibration_package_contract_writer
    - out_contract
    - stage2_calibration_package_contract_validator
  extra_nodes:
  - id: in_stage1_contract_s2
    label: dataset_build_output.json
    node_type: artifact
    description: Stage 1 handoff contract preferred for Stage 2 input resolution
  - id: in_cps_s5
    label: source_imputed_stratified_extended_cps.h5
    node_type: artifact
    description: From Stage 4 (~12K HH)
  - id: in_db_s5
    label: policy_data.db
    node_type: external
    description: SQLite calibration database - 10-step ETL
  - id: in_config_s5
    label: target_config.yaml
    node_type: artifact
    description: Active target include list
  - id: in_blocks_s5
    label: block_cd_distributions.csv.gz
    node_type: artifact
    description: Census block populations
  - id: target_resolve
    label: Target Resolution
    node_type: process
    description: SQL query to target_overview - ~8,000 rows total
  - id: target_uprate
    label: Target Uprating
    node_type: process
    description: CPI-U for dollars, pop growth for counts
  - id: geo_build
    label: Geography Index Build
    node_type: process
    description: state_to_cols, cd_to_cols maps
  - id: constraint_resolve
    label: Constraint Resolution
    node_type: process
    description: Non-geographic constraints from DB (age, medicaid, filer)
  - id: takeup_rerand
    label: Block-Level Takeup Re-randomization
    node_type: process
    description: Seeded on (block_geoid, hh_id) - ensures calibration consistency
  - id: sparse_build
    label: Sparse Matrix Construction
    node_type: process
    description: COO triples -> CSR matrix - shape (n_targets, 5.16M), ~0.02% nonzero
  - id: out_pkg
    label: calibration_package.pkl
    node_type: artifact
    description: X_sparse CSR matrix, targets_df, initial_weights, metadata
  - id: out_metadata
    label: calibration_package_meta.json
    node_type: artifact
    description: Metadata sidecar generated from the typed package payload and Stage 2 contract
  - id: out_targets
    label: calibration_targets.jsonl
    node_type: artifact
    description: Row-level selected target metadata with stable target_id and target_index join keys
  - id: out_target_facets
    label: calibration_target_facets.json
    node_type: artifact
    description: Compact target counts by variable, geography level, target name, period, and constraint key
  - id: out_geography_summary
    label: geography_assignment_summary.json
    node_type: artifact
    description: Compact clone-major geography identity summary for block, county, state, and congressional district arrays
  - id: out_contract
    label: calibration_package_contract.json
    node_type: artifact
    description: Stage 2 package handoff contract written next to calibration_package.pkl
  - id: util_sql
    label: sqlalchemy
    node_type: utility
    description: Database queries
  - id: util_pool
    label: ProcessPoolExecutor
    node_type: utility
    description: Parallel state computation
  - id: util_takeup_s5
    label: compute_block_takeup_for_entities()
    node_type: utility
    description: utils/takeup.py
  - id: util_scipy
    label: scipy.sparse
    node_type: utility
    description: CSR/COO matrix construction
  edges:
  - source: in_stage1_contract_s2
    target: stage2_input_bundle
    edge_type: data_flow
    label: preferred input contract
  - source: in_cps_s5
    target: stage2_input_bundle
    edge_type: data_flow
    label: compatibility fallback
  - source: in_db_s5
    target: stage2_input_bundle
    edge_type: external_source
    label: compatibility fallback
  - source: stage2_input_bundle
    target: stage2_build_context
    edge_type: data_flow
    label: validated inputs
  - source: stage2_artifact_specs
    target: stage2_build_context
    edge_type: uses_utility
    label: output bundle paths
  - source: stage2_build_context
    target: target_resolve
    edge_type: data_flow
    label: dataset and database paths
  - source: stage2_build_context
    target: stage2_calibration_package_writer
    edge_type: uses_utility
    label: package output bundle
  - source: in_db_s5
    target: target_resolve
    edge_type: external_source
    label: SQL targets
  - source: in_config_s5
    target: stage2_target_config_identity
    edge_type: data_flow
    label: config file
  - source: stage2_target_config_identity
    target: stage2_target_catalog_load
    edge_type: data_flow
    label: resolved path and checksum
  - source: in_db_s5
    target: stage2_target_catalog_reader
    edge_type: external_source
    label: active and disabled targets
  - source: stage2_target_catalog_reader
    target: stage2_target_selection_policy
    edge_type: data_flow
    label: target catalog
  - source: stage2_target_catalog_load
    target: stage2_target_selection_policy
    edge_type: data_flow
    label: include/exclude rules
  - source: stage2_target_selection_policy
    target: stage2_target_selection_result
    edge_type: data_flow
    label: selected targets
  - source: stage2_target_selection_result
    target: build_matrix
    edge_type: data_flow
    label: matrix target order
  - source: stage2_target_selection_result
    target: build_matrix_chunked
    edge_type: data_flow
    label: matrix target order
  - source: in_cps_s5
    target: stage2_geography_assignment_spec
    edge_type: data_flow
    label: household AGI and fixed state overrides
  - source: in_db_s5
    target: stage2_geography_assignment_spec
    edge_type: external_source
    label: district AGI targets
  - source: stage2_geography_assignment_spec
    target: stage2_geography_assignment_result
    edge_type: data_flow
    label: deterministic assignment inputs
  - source: in_blocks_s5
    target: stage2_geography_assignment_result
    edge_type: data_flow
    label: block populations
  - source: stage2_geography_assignment_result
    target: build_matrix
    edge_type: data_flow
    label: clone-major geography
  - source: stage2_geography_assignment_result
    target: build_matrix_chunked
    edge_type: data_flow
    label: clone-major geography
  - source: stage2_target_catalog_load
    target: stage2_target_config_apply
    edge_type: data_flow
    label: include/exclude rules
  - source: target_resolve
    target: stage2_target_config_apply
    edge_type: data_flow
    label: candidate targets
  - source: stage2_target_config_apply
    target: target_uprate
    edge_type: data_flow
    label: selected targets
  - source: target_uprate
    target: geo_build
    edge_type: data_flow
  - source: geo_build
    target: constraint_resolve
    edge_type: data_flow
  - source: constraint_resolve
    target: state_precomp
    edge_type: data_flow
  - source: in_cps_s5
    target: state_precomp
    edge_type: data_flow
    label: household data
  - source: state_precomp
    target: clone_assembly
    edge_type: data_flow
  - source: in_blocks_s5
    target: clone_assembly
    edge_type: data_flow
    label: block populations
  - source: clone_assembly
    target: takeup_rerand
    edge_type: data_flow
  - source: takeup_rerand
    target: sparse_build
    edge_type: data_flow
  - source: sparse_build
    target: build_matrix
    edge_type: uses_library
    label: non-chunked path
  - source: sparse_build
    target: build_matrix_chunked
    edge_type: uses_library
    label: chunked path
  - source: build_matrix
    target: stage2_payload_boundary
    edge_type: data_flow
  - source: build_matrix_chunked
    target: stage2_payload_boundary
    edge_type: data_flow
  - source: stage2_payload_boundary
    target: stage2_payload_writer
    edge_type: data_flow
    label: typed pickle payload
  - source: stage2_payload_writer
    target: stage2_calibration_package_writer
    edge_type: uses_library
    label: pickle write
  - source: stage2_artifact_specs
    target: stage2_calibration_package_writer
    edge_type: uses_utility
    label: package path
  - source: stage2_calibration_package_writer
    target: out_pkg
    edge_type: produces_artifact
  - source: stage2_payload_writer
    target: out_metadata
    edge_type: produces_artifact
    label: sidecar metadata
  - source: stage2_target_selection_result
    target: out_targets
    edge_type: produces_artifact
    label: row-level selected targets
  - source: stage2_target_selection_result
    target: out_target_facets
    edge_type: produces_artifact
    label: derived facets
  - source: stage2_geography_assignment_result
    target: out_geography_summary
    edge_type: produces_artifact
    label: geography identity
  - source: out_pkg
    target: stage2_payload_reader
    edge_type: data_flow
  - source: out_pkg
    target: stage2_calibration_package_contract_writer
    edge_type: data_flow
  - source: stage2_payload_reader
    target: stage2_calibration_package_contract_writer
    edge_type: uses_library
    label: summary and checksum
  - source: out_targets
    target: stage2_calibration_package_contract_writer
    edge_type: data_flow
    label: target metadata artifact
  - source: out_target_facets
    target: stage2_calibration_package_contract_writer
    edge_type: data_flow
    label: target facet artifact
  - source: out_geography_summary
    target: stage2_calibration_package_contract_writer
    edge_type: data_flow
    label: geography summary artifact
  - source: stage2_artifact_specs
    target: stage2_calibration_package_contract_writer
    edge_type: uses_utility
    label: contract path
  - source: stage2_calibration_package_contract_writer
    target: out_contract
    edge_type: produces_artifact
  - source: stage2_calibration_package_contract_writer
    target: out_targets
    edge_type: validates
  - source: stage2_calibration_package_contract_writer
    target: out_target_facets
    edge_type: validates
  - source: stage2_calibration_package_contract_writer
    target: out_geography_summary
    edge_type: validates
  - source: out_pkg
    target: stage2_calibration_package_contract_validator
    edge_type: validates
  - source: out_contract
    target: stage2_calibration_package_contract_validator
    edge_type: validates
  - source: in_cps_s5
    target: stage2_calibration_package_contract_validator
    edge_type: validates
  - source: in_db_s5
    target: stage2_calibration_package_contract_validator
    edge_type: validates
  - source: util_sql
    target: target_resolve
    edge_type: uses_utility
  - source: util_pool
    target: state_precomp
    edge_type: uses_utility
  - source: util_takeup_s5
    target: takeup_rerand
    edge_type: uses_utility
  - source: util_scipy
    target: sparse_build
    edge_type: uses_utility
  status: current
  stability: moving
- id: 3a_weight_fitting_regional
  label: 3a
  title: 'Substage 3a: Weight Fitting - Regional'
  canonical_stage_id: 3_fit_weights
  legacy_stage_id: '6'
  manifest_step_ids:
  - 03_fit_weights_regional
  description: Fit regional log-weights using L0 HardConcrete gates on GPU
  country: us
  groups:
  - id: run_calibration_fit
    label: run_calibration()
    description: 'Fit phase: initialize weights, optimize sparse calibration weights, and emit artifacts plus diagnostics'
    node_ids:
    - init_weights
    - create_model
    - fit_model
    - extract_weights
    - out_weights
    - out_geo_s6
    - out_diag
    - out_config_s6
  extra_nodes:
  - id: in_pkg_s6
    label: calibration_package.pkl
    node_type: artifact
    description: From Stage 5 - X_sparse, targets_df, initial_weights
  - id: modal_gpu
    label: Modal GPU Container
    node_type: external
    description: T4 / A10 / A100 / H100 - 32GB RAM, 8 CPU
  - id: create_model
    label: Create SparseCalibrationWeights
    node_type: process
    description: n_features = 5.16M, init_keep_prob = 0.999
  - id: extract_weights
    label: Extract Weights
    node_type: process
    description: Deterministic gate threshold - produces exact zeros
  - id: out_weights
    label: calibration_weights.npy
    node_type: artifact
    description: 'Shape: (n_records x n_clones) - most entries zero'
  - id: out_geo_s6
    label: geography.npz
    node_type: artifact
    description: block_geoid, cd_geoid, county_fips, state_fips
  - id: out_diag
    label: unified_diagnostics.csv
    node_type: artifact
    description: Per-target error analysis
  - id: out_config_s6
    label: unified_run_config.json
    node_type: artifact
    description: Hyperparameters + SHA256 checksums
  - id: util_l0
    label: l0-python
    node_type: utility
    description: SparseCalibrationWeights - HardConcrete gates
  - id: util_pytorch
    label: PyTorch
    node_type: utility
    description: Adam optimizer, autograd, CUDA
  edges:
  - source: in_pkg_s6
    target: init_weights
    edge_type: data_flow
  - source: init_weights
    target: create_model
    edge_type: data_flow
  - source: create_model
    target: fit_model
    edge_type: data_flow
  - source: modal_gpu
    target: fit_model
    edge_type: runs_on_infra
    label: runs on
  - source: fit_model
    target: extract_weights
    edge_type: data_flow
  - source: extract_weights
    target: out_weights
    edge_type: produces_artifact
  - source: extract_weights
    target: out_geo_s6
    edge_type: produces_artifact
  - source: fit_model
    target: out_diag
    edge_type: produces_artifact
  - source: fit_model
    target: out_config_s6
    edge_type: produces_artifact
  - source: util_l0
    target: create_model
    edge_type: uses_utility
  - source: util_pytorch
    target: fit_model
    edge_type: uses_utility
  status: current
  stability: moving
- id: 3b_weight_fitting_national
  label: 3b
  title: 'Substage 3b: Weight Fitting - National'
  canonical_stage_id: 3_fit_weights
  legacy_stage_id: '6'
  manifest_step_ids:
  - 03_fit_weights_national
  description: Fit national log-weights for the national H5 output using the same L0 calibration machinery
  country: us
  groups:
  - id: run_calibration_fit_national
    label: run_calibration() national
    description: 'National fit phase: initialize national weights, optimize sparse calibration weights, and emit national artifacts plus diagnostics'
    node_ids:
    - init_weights
    - create_model_national
    - fit_model
    - extract_national_weights
    - out_national_weights
    - out_national_geo_s6
    - out_national_diag
    - out_national_config_s6
  extra_nodes:
  - id: in_pkg_national_s6
    label: calibration_package.pkl
    node_type: artifact
    description: From substage 2a - X_sparse, targets_df, initial_weights
  - id: modal_gpu_national
    label: Modal GPU Container
    node_type: external
    description: T4 / A10 / A100 / H100 for national calibration
  - id: create_model_national
    label: Create National SparseCalibrationWeights
    node_type: process
    description: National target scope with L0 HardConcrete gates
  - id: extract_national_weights
    label: Extract National Weights
    node_type: process
    description: Deterministic gate threshold for national output weights
  - id: out_national_weights
    label: national_calibration_weights.npy
    node_type: artifact
    description: National weights used for US.h5
  - id: out_national_geo_s6
    label: national_geography_assignment.npz
    node_type: artifact
    description: National geography assignment artifact
  - id: out_national_diag
    label: national_unified_diagnostics.csv
    node_type: artifact
    description: Per-target national error analysis
  - id: out_national_config_s6
    label: national_unified_run_config.json
    node_type: artifact
    description: National hyperparameters + SHA256 checksums
  - id: util_l0_national
    label: l0-python
    node_type: utility
    description: SparseCalibrationWeights - HardConcrete gates
  - id: util_pytorch_national
    label: PyTorch
    node_type: utility
    description: Adam optimizer, autograd, CUDA
  edges:
  - source: in_pkg_national_s6
    target: init_weights
    edge_type: data_flow
  - source: init_weights
    target: create_model_national
    edge_type: data_flow
  - source: create_model_national
    target: fit_model
    edge_type: data_flow
  - source: modal_gpu_national
    target: fit_model
    edge_type: runs_on_infra
    label: runs on
  - source: fit_model
    target: extract_national_weights
    edge_type: data_flow
  - source: extract_national_weights
    target: out_national_weights
    edge_type: produces_artifact
  - source: extract_national_weights
    target: out_national_geo_s6
    edge_type: produces_artifact
  - source: fit_model
    target: out_national_diag
    edge_type: produces_artifact
  - source: fit_model
    target: out_national_config_s6
    edge_type: produces_artifact
  - source: util_l0_national
    target: create_model_national
    edge_type: uses_utility
  - source: util_pytorch_national
    target: fit_model
    edge_type: uses_utility
  status: current
  stability: moving
- id: 4a_local_area_h5_regional
  label: 4a
  title: 'Substage 4a: Local Area H5 - Regional'
  canonical_stage_id: 4_build_outputs
  legacy_stage_id: '7'
  manifest_step_ids:
  - 04_build_h5_regional
  description: Build 51 state + 435 district + 1 city H5 files on Modal workers
  country: us
  extra_nodes:
  - id: in_weights_s7
    label: calibration_weights.npy
    node_type: artifact
    description: From Stage 6
  - id: in_dataset_s7
    label: source_imputed_stratified_extended_cps.h5
    node_type: artifact
    description: From Stage 4
  - id: in_db_s7
    label: policy_data.db
    node_type: external
    description: CD list for partitioning
  - id: modal_coord
    label: Modal Coordinator
    node_type: external
    description: coordinate_publish() - 8GB RAM, 24hr timeout
  - id: partition
    label: Partition Work
    node_type: process
    description: '50 workers - weights: state by CD count, city=11, district=1'
  - id: worker_s7
    label: Modal Worker Container
    node_type: external
    description: 16GB RAM, 1 CPU each, 8-hour timeout
  - id: takeup_apply
    label: Takeup Re-application
    node_type: process
    description: 10 takeup variables - block-level seeded draws
  - id: out_states
    label: states/*.h5
    node_type: artifact
    description: 51 files - AL.h5 through WY.h5
  - id: out_districts
    label: districts/*.h5
    node_type: artifact
    description: ~435 files - NC-01.h5, CA-52.h5, ...
  - id: out_cities
    label: cities/*.h5
    node_type: artifact
    description: 1 file - NYC.h5
  - id: out_manifest
    label: manifest.json
    node_type: artifact
    description: SHA256 checksums for all H5 files
  - id: util_build_h5
    label: publish_local_area.build_h5()
    node_type: utility
    description: calibration/publish_local_area.py
  - id: util_takeup_s7
    label: apply_block_takeup_to_arrays()
    node_type: utility
    description: utils/takeup.py
  edges:
  - source: in_weights_s7
    target: partition
    edge_type: data_flow
  - source: in_dataset_s7
    target: partition
    edge_type: data_flow
  - source: in_db_s7
    target: partition
    edge_type: external_source
    label: CD list
  - source: partition
    target: build_states
    edge_type: data_flow
  - source: build_states
    target: build_districts
    edge_type: data_flow
  - source: build_districts
    target: build_cities
    edge_type: data_flow
  - source: build_states
    target: build_h5
    edge_type: data_flow
    label: calls
  - source: build_districts
    target: build_h5
    edge_type: data_flow
    label: calls
  - source: build_cities
    target: build_h5
    edge_type: data_flow
    label: calls
  - source: build_h5
    target: geo_derive
    edge_type: data_flow
  - source: geo_derive
    target: takeup_apply
    edge_type: data_flow
  - source: modal_coord
    target: worker_s7
    edge_type: runs_on_infra
    label: orchestrates
  - source: worker_s7
    target: build_h5
    edge_type: runs_on_infra
    label: runs
  - source: build_states
    target: out_states
    edge_type: produces_artifact
  - source: build_districts
    target: out_districts
    edge_type: produces_artifact
  - source: build_cities
    target: out_cities
    edge_type: produces_artifact
  - source: build_h5
    target: out_manifest
    edge_type: produces_artifact
  - source: util_build_h5
    target: build_h5
    edge_type: uses_utility
  - source: util_takeup_s7
    target: takeup_apply
    edge_type: uses_utility
  status: current
  stability: moving
- id: 4b_local_area_h5_national
  label: 4b
  title: 'Substage 4b: Local Area H5 - National'
  canonical_stage_id: 4_build_outputs
  legacy_stage_id: '7'
  manifest_step_ids:
  - 04_build_h5_national
  description: Build the national US.h5 output from national weights and national geography artifacts
  country: us
  extra_nodes:
  - id: in_national_weights_s4b
    label: national_calibration_weights.npy
    node_type: artifact
    description: From substage 3b
  - id: in_national_dataset_s4b
    label: source_imputed_stratified_extended_cps.h5
    node_type: artifact
    description: From substage 1f
  - id: in_national_geo_s4b
    label: national_geography_assignment.npz
    node_type: artifact
    description: From substage 3b
  - id: in_national_config_s4b
    label: national_unified_run_config.json
    node_type: artifact
    description: From substage 3b
  - id: national_h5_coord
    label: National H5 Coordinator
    node_type: process
    description: coordinate_national_publish() prepares the US request and worker inputs
  - id: national_worker
    label: National Modal Worker
    node_type: external
    description: Single worker build for national/US.h5
  - id: national_request
    label: AreaBuildRequest(type=national)
    node_type: process
    description: Typed request for the national US.h5 output
  - id: national_validation
    label: National H5 Validation
    node_type: process
    description: validate_national_h5.py structural and reference-value checks
  - id: out_national_h5
    label: national/US.h5
    node_type: artifact
    description: National H5 artifact staged under the run ID
  - id: out_national_validation
    label: national_validation.txt
    node_type: artifact
    description: Captured national validation output
  - id: util_build_h5_national
    label: publish_local_area.build_h5()
    node_type: utility
    description: Shared H5 writer used by regional and national outputs
  edges:
  - source: in_national_weights_s4b
    target: national_request
    edge_type: data_flow
  - source: in_national_dataset_s4b
    target: national_request
    edge_type: data_flow
  - source: in_national_geo_s4b
    target: national_request
    edge_type: data_flow
  - source: in_national_config_s4b
    target: national_request
    edge_type: data_flow
  - source: national_request
    target: build_h5
    edge_type: data_flow
  - source: national_h5_coord
    target: national_worker
    edge_type: runs_on_infra
    label: spawns
  - source: national_worker
    target: build_h5
    edge_type: runs_on_infra
    label: runs
  - source: build_h5
    target: out_national_h5
    edge_type: produces_artifact
  - source: out_national_h5
    target: national_validation
    edge_type: data_flow
  - source: national_validation
    target: out_national_validation
    edge_type: produces_artifact
  - source: util_build_h5_national
    target: build_h5
    edge_type: uses_utility
  status: current
  stability: moving
- id: 4d_upload_diagnostics
  label: 4d
  title: 'Substage 4d: Upload Diagnostics'
  canonical_stage_id: 4_build_outputs
  legacy_stage_id: '7'
  manifest_step_ids:
  - 04_upload_diagnostics
  description: Collect calibration and validation diagnostics and upload them to run-scoped archival paths
  country: us
  extra_nodes:
  - id: in_calibration_diag_s4d
    label: unified_diagnostics.csv
    node_type: artifact
    description: Calibration diagnostics emitted by substage 3a/3b
  - id: in_validation_diag_s4d
    label: validation_results.csv / national_validation.txt
    node_type: artifact
    description: H5 validation diagnostics emitted by substage 4a/4b
  - id: upload_diagnostics_s4d
    label: Upload Run Diagnostics
    node_type: process
    description: upload_run_diagnostics() writes diagnostics to HuggingFace archival paths
  - id: out_hf_diagnostics_s4d
    label: calibration/runs/{run_id}/diagnostics/
    node_type: external
    description: Run-scoped diagnostics archive
  edges:
  - source: in_calibration_diag_s4d
    target: upload_diagnostics_s4d
    edge_type: data_flow
  - source: in_validation_diag_s4d
    target: upload_diagnostics_s4d
    edge_type: data_flow
  - source: calibration_diagnostics
    target: in_calibration_diag_s4d
    edge_type: produces_artifact
  - source: upload_diagnostics_s4d
    target: out_hf_diagnostics_s4d
    edge_type: produces_artifact
  status: current
  stability: moving
- id: 5a_validate_outputs
  label: 5a
  title: 'Substage 5a: Validate Outputs'
  canonical_stage_id: 5_validate_and_promote_release
  legacy_stage_id: '8'
  manifest_step_ids:
  - 05_promote_release
  description: Validate staged H5 and base artifacts before any production promotion
  country: us
  extra_nodes:
  - id: in_h5s
    label: 51 state + 435 district + 1 city H5s
    node_type: artifact
    description: From Stage 7 (Modal volume)
  - id: in_db_s8
    label: policy_data.db
    node_type: external
    description: Validation targets
  - id: v1
    label: 'Layer 1: Manifest Verification'
    node_type: process
    description: SHA256 checksums per file - catches corruption
  - id: v4
    label: 'Layer 4: Smoke Test'
    node_type: process
    description: Sum 20 key vars across 51 states - GDP ~$29T, pop ~335M
  - id: v5
    label: 'Layer 5: National H5 Validation'
    node_type: process
    description: '>30% deviation flagged - hardcoded reference values'
  - id: v6
    label: 'Layer 6: Pre-Upload Validation'
    node_type: process
    description: File size minimums (100MB/50MB), H5 structure, Microsim aggregates
  - id: v7
    label: 'Layer 7: Package Validation'
    node_type: process
    description: Matrix achievability, target ratio analysis, zero-row detection
  - id: out_validated_candidates_s5a
    label: validated release candidates
    node_type: artifact
    description: Manifest-backed artifacts that passed validation and are eligible for promotion
  - id: util_manifest_s8
    label: manifest.py
    node_type: utility
    description: generate_manifest(), verify_manifest()
  - id: util_sanity
    label: sanity_checks.py
    node_type: utility
    description: run_sanity_checks()
  - id: util_validate
    label: validate_staging.py
    node_type: utility
    description: Full target comparison
  edges:
  - source: in_h5s
    target: v1
    edge_type: data_flow
  - source: in_db_s8
    target: target_validation
    edge_type: external_source
    label: targets
  - source: v1
    target: sanity_checks
    edge_type: data_flow
  - source: sanity_checks
    target: target_validation
    edge_type: data_flow
  - source: target_validation
    target: v4
    edge_type: data_flow
  - source: v4
    target: v5
    edge_type: data_flow
  - source: v5
    target: v6
    edge_type: data_flow
  - source: v6
    target: v7
    edge_type: data_flow
  - source: v7
    target: out_validated_candidates_s5a
    edge_type: produces_artifact
    label: all pass
  - source: util_manifest_s8
    target: v1
    edge_type: uses_utility
  - source: util_sanity
    target: sanity_checks
    edge_type: uses_utility
  - source: util_validate
    target: target_validation
    edge_type: uses_utility
  status: current
  stability: moving
- id: 5b_promote_huggingface
  label: 5b
  title: 'Substage 5b: Promote HuggingFace'
  canonical_stage_id: 5_validate_and_promote_release
  legacy_stage_id: '8'
  manifest_step_ids:
  - 05_promote_release
  description: Promote validated staged artifacts to HuggingFace production paths
  country: us
  extra_nodes:
  - id: in_validated_candidates_s5b
    label: validated release candidates
    node_type: artifact
    description: Output set from substage 5a
  - id: hf_staging_s5b
    label: HuggingFace staging/{candidate_version}/{run_id}
    node_type: external
    description: Run-scoped staging prefix containing validated artifacts
  - id: out_hf_prod
    label: HuggingFace Production
    node_type: external
    description: policyengine/policyengine-us-data - final published datasets
  - id: util_upload_s5b
    label: data_upload.py
    node_type: utility
    description: staging/promote helpers
  edges:
  - source: in_validated_candidates_s5b
    target: staging_upload
    edge_type: data_flow
  - source: hf_staging_s5b
    target: atomic_promote
    edge_type: external_source
  - source: staging_upload
    target: atomic_promote
    edge_type: data_flow
  - source: promote_pipeline_run
    target: atomic_promote
    edge_type: data_flow
    label: orchestrates
  - source: atomic_promote
    target: out_hf_prod
    edge_type: produces_artifact
  - source: util_upload_s5b
    target: staging_upload
    edge_type: uses_utility
  - source: util_upload_s5b
    target: atomic_promote
    edge_type: uses_utility
  status: current
  stability: moving
- id: 5c_promote_gcs
  label: 5c
  title: 'Substage 5c: Promote GCS'
  canonical_stage_id: 5_validate_and_promote_release
  legacy_stage_id: '8'
  manifest_step_ids:
  - 05_promote_release
  description: Upload promoted datasets to Google Cloud Storage with version metadata
  country: us
  extra_nodes:
  - id: in_hf_prod_s5c
    label: HuggingFace production release
    node_type: external
    description: Production artifacts promoted in substage 5b
  - id: gcs_upload
    label: GCS Parallel Upload
    node_type: process
    description: gs://policyengine-us-data/ - version metadata on each blob
  - id: out_gcs
    label: Google Cloud Storage
    node_type: external
    description: gs://policyengine-us-data/ - CDN/backup
  - id: util_upload_gcs_s5c
    label: data_upload.py
    node_type: utility
    description: upload_from_hf_staging_to_gcs() and related GCS helpers
  edges:
  - source: in_hf_prod_s5c
    target: gcs_upload
    edge_type: data_flow
  - source: atomic_promote
    target: gcs_upload
    edge_type: data_flow
    label: release files
  - source: gcs_upload
    target: out_gcs
    edge_type: produces_artifact
  - source: util_upload_gcs_s5c
    target: gcs_upload
    edge_type: uses_utility
  status: current
  stability: moving
- id: 5d_write_version_manifest
  label: 5d
  title: 'Substage 5d: Write Version Manifest'
  canonical_stage_id: 5_validate_and_promote_release
  legacy_stage_id: '8'
  manifest_step_ids:
  - 05_promote_release
  description: Finalize release manifests, record run diagnostics paths, and clean staging state
  country: us
  extra_nodes:
  - id: in_release_outputs_s5d
    label: promoted release outputs
    node_type: artifact
    description: HuggingFace and GCS artifacts from substages 5b and 5c
  - id: version_manifest_write
    label: version_manifest.json update
    node_type: process
    description: Register the version, run ID, release artifacts, and diagnostics path
  - id: staging_cleanup
    label: Staging Cleanup
    node_type: process
    description: cleanup_staging_hf() - removes run-scoped staging files when release is finalized
  - id: out_version_manifest
    label: version_manifest.json
    node_type: artifact
    description: Durable release registry for legacy consumers
  - id: out_release_finalized
    label: finalized release
    node_type: artifact
    description: Release has production artifacts, manifest identity, and cleaned staging state
  - id: util_manifest_s5d
    label: version_manifest.py / release_manifest.py
    node_type: utility
    description: Version and release manifest construction helpers
  edges:
  - source: in_release_outputs_s5d
    target: version_manifest_write
    edge_type: data_flow
  - source: atomic_promote
    target: version_manifest_write
    edge_type: data_flow
    label: release manifest inputs
  - source: version_manifest_write
    target: out_version_manifest
    edge_type: produces_artifact
  - source: version_manifest_write
    target: staging_cleanup
    edge_type: data_flow
  - source: staging_cleanup
    target: out_release_finalized
    edge_type: produces_artifact
  - source: util_manifest_s5d
    target: version_manifest_write
    edge_type: uses_utility
  status: current
  stability: moving