Merge pull request #176 from PolicyEngine/single-year-dataset

Migrate to new single-year dataset format for efficiency

Author: nikhilwoodruff

.github/workflows/push.yaml

@@ -70,10 +70,3 @@ jobs:
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       - name: Publish a git tag
         run: ".github/publish-git-tag.sh || true"
-      - name: Publish a Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI }}
-          skip-existing: true
-          verbose: true

Makefile

@@ -12,8 +12,7 @@ install:
 	pip install -e ".[dev]" --config-settings editable_mode=compat
 
 install-uv:
-	uv pip install --system policyengine-uk
-	uv pip install --system policyengine>=2.40.2
+	uv pip install --system "jupyter-book>=2.0.0a0"
 	uv pip install --system -e ".[dev]" --config-settings editable_mode=compat
 
 download:
@@ -37,6 +36,7 @@ data:
 	python policyengine_uk_data/datasets/frs/local_areas/constituencies/calibrate.py
 	python policyengine_uk_data/datasets/frs/local_areas/local_authorities/calibrate.py
 	python policyengine_uk_data/utils/create_multi_year_dataset.py
+	python policyengine_uk_data/storage/migrate_to_uk_single_year_datasets.py
 
 efrs:
 	python policyengine_uk_data/datasets/frs/enhanced_frs.py

changelog_entry.yaml

@@ -0,0 +1,4 @@
+- bump: patch
+  changes:
+    fixed:
+    - Migrated to more efficient dataset version.

policyengine_uk_data/storage/migrate_to_uk_single_year_datasets.py

@@ -0,0 +1,19 @@
+from policyengine_uk.data import UKSingleYearDataset
+from policyengine_uk import Microsimulation
+from policyengine_core.data import Dataset
+from policyengine_uk_data.datasets import EnhancedFRS_2023_24, FRS_2023_24
+
+
+def migrate_to_uk_single_year_dataset(file_path: str):
+    sim = Microsimulation(dataset=Dataset.from_file(file_path))
+
+    single_year_dataset = UKSingleYearDataset.from_simulation(
+        sim, fiscal_year=2023
+    )
+
+    single_year_dataset.save(file_path)
+
+
+if __name__ == "__main__":
+    migrate_to_uk_single_year_dataset(FRS_2023_24.file_path)
+    migrate_to_uk_single_year_dataset(EnhancedFRS_2023_24.file_path)

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
     "google-cloud-storage",
     "google-auth",
     "uk-public-services-imputation",
-    "policyengine-uk>=2.40.2",
+    "policyengine-uk==2.40.2",
 ]
 
 [project.optional-dependencies]
@@ -32,7 +32,6 @@ dev = [
     "torch",
     "tables",
     "furo",
-    "jupyter-book",
     "yaml-changelog>=0.1.7",
     "itables",
     "quantile-forest",