Update LCFS, WAS, and ETB prerequisites (#413)

* Update private UK survey prerequisites

* Use current UKDS tab zip layouts

* Handle refreshed LCFS and ETB aggregate shifts

* Use locked uv environment in data CI

Author: MaxGhenis

.github/workflows/pull_request.yaml

@@ -64,14 +64,14 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: 3.13
-      - name: Install package
-        run: uv pip install -e ".[dev]" --system
+      - name: Sync locked environment
+        run: uv sync --frozen --all-extras
       - name: Download data inputs
-        run: make download
+        run: uv run --frozen make download
         env:
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       - name: Build datasets
-        run: make data
+        run: uv run --frozen make data
         env:
           TESTING: "1"
       - name: Save calibration log (constituencies)
@@ -87,4 +87,4 @@ jobs:
           path: la_calibration_log.csv
 
       - name: Run tests
-        run: make test
+        run: uv run --frozen make test

.github/workflows/push.yaml

@@ -52,14 +52,14 @@ jobs:
         with:
           workload_identity_provider: "projects/322898545428/locations/global/workloadIdentityPools/policyengine-research-id-pool/providers/prod-github-provider"
           service_account: "policyengine-research@policyengine-research.iam.gserviceaccount.com"
-      - name: Install package
-        run: uv pip install -e ".[dev]" --system
+      - name: Sync locked environment
+        run: uv sync --frozen --all-extras
       - name: Download data inputs
-        run: make download
+        run: uv run --frozen make download
         env:
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       - name: Build datasets
-        run: make data
+        run: uv run --frozen make data
       - name: Save calibration log (constituencies)
         uses: actions/upload-artifact@v7
         with:
@@ -72,9 +72,9 @@ jobs:
           name: la_calibration_log.csv
           path: la_calibration_log.csv
       - name: Run tests
-        run: make test
+        run: uv run --frozen make test
       - name: Upload data
-        run: make upload
+        run: uv run --frozen make upload
         env:
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       - name: Publish a git tag

policyengine_uk_data/datasets/create_datasets.py

@@ -103,6 +103,7 @@ def main():
             "Clone and assign OA geography",
             "Calibrate constituency weights",
             "Calibrate local authority weights",
+            "Calibrate public service aggregates",
             "Calibrate fuel litres",
             "Save final dataset",
             "Create tiny datasets",
@@ -288,6 +289,17 @@ def main():
                 )
                 update_dataset(materialize_step, "completed")
 
+            update_dataset("Calibrate public service aggregates", "processing")
+            from policyengine_uk_data.datasets.imputations.services.services import (
+                calibrate_rail_subsidy_spending,
+            )
+
+            calibrate_rail_subsidy_spending(
+                frs_calibrated,
+                frs_release.calibration_year,
+            )
+            update_dataset("Calibrate public service aggregates", "completed")
+
             update_dataset("Calibrate fuel litres", "processing")
             from policyengine_uk_data.datasets.imputations.consumption import (
                 calibrate_dataset_fuel_litre_proxies_to_road_fuel,