Use current UKDS tab zip layouts

Author: MaxGhenis

policyengine_uk_data/datasets/private_releases.py

@@ -8,6 +8,7 @@ class LCFSRelease:
     fuel_price_year: int
     ukds_study_number: int
     doi: str
+    ukds_tab_subdir: str
     household_tab_filename: str
     person_tab_filename: str
 
@@ -23,6 +24,7 @@ class WASRelease:
     end_year: int
     ukds_study_number: int
     doi: str
+    ukds_tab_subdir: str
     household_tab_filename: str
     person_tab_filename: str
 
@@ -38,6 +40,7 @@ class ETBRelease:
     default_training_year: int
     ukds_study_number: int
     doi: str
+    ukds_tab_subdir: str
     household_tab_filename: str
     person_tab_filename: str
 
@@ -52,8 +55,9 @@ def raw_zip_name(self) -> str:
     fuel_price_year=2023,
     ukds_study_number=9468,
     doi="10.5255/UKDA-SN-9468-3",
-    household_tab_filename="9468_dvhh_ukanon_v2_2023.tab",
-    person_tab_filename="9468_dvper_ukanon_202324_2023.tab",
+    ukds_tab_subdir="UKDA-9468-tab/tab",
+    household_tab_filename="dvhh_ukanon_v2_2023.tab",
+    person_tab_filename="dvper_ukanon_202324_2023.tab",
 )
 
 
@@ -63,8 +67,9 @@ def raw_zip_name(self) -> str:
     end_year=2022,
     ukds_study_number=7215,
     doi="10.5255/UKDA-SN-7215-20",
-    household_tab_filename="7215_was_round_8_hhold_eul_may_2025_230525.tab",
-    person_tab_filename="7215_was_round_8_person_eul_may_2025_230525.tab",
+    ukds_tab_subdir="UKDA-7215-tab/tab",
+    household_tab_filename="was_round_8_hhold_eul_may_2025_230525.tab",
+    person_tab_filename="was_round_8_person_eul_may_2025_230525.tab",
 )
 
 
@@ -74,6 +79,7 @@ def raw_zip_name(self) -> str:
     default_training_year=2023,
     ukds_study_number=8856,
     doi="10.5255/UKDA-SN-8856-4",
-    household_tab_filename="8856_householdv2_1977-2024.tab",
-    person_tab_filename="8856_personv2_2018-2024.tab",
+    ukds_tab_subdir="UKDA-8856-tab/tab",
+    household_tab_filename="householdv2_1977-2024.tab",
+    person_tab_filename="personv2_2018-2024.tab",
 )

policyengine_uk_data/storage/download_private_prerequisites.py

@@ -16,9 +16,9 @@
 
 PRIVATE_PREREQUISITES = [
     (CURRENT_FRS_RELEASE.raw_zip_name, CURRENT_FRS_RELEASE.ukds_tab_subdir),
-    (CURRENT_LCFS_RELEASE.raw_zip_name, None),
-    (CURRENT_WAS_RELEASE.raw_zip_name, None),
-    (CURRENT_ETB_RELEASE.raw_zip_name, None),
+    (CURRENT_LCFS_RELEASE.raw_zip_name, CURRENT_LCFS_RELEASE.ukds_tab_subdir),
+    (CURRENT_WAS_RELEASE.raw_zip_name, CURRENT_WAS_RELEASE.ukds_tab_subdir),
+    (CURRENT_ETB_RELEASE.raw_zip_name, CURRENT_ETB_RELEASE.ukds_tab_subdir),
     (f"{SPI_RELEASE_NAME}.zip", None),
 ]
 

policyengine_uk_data/tests/test_frs_prerequisites.py

@@ -37,24 +37,30 @@ def test_private_prerequisites_use_current_spi_release():
 
 
 def test_private_prerequisites_use_current_lcfs_release():
-    prerequisite_names = [filename for filename, _ in PRIVATE_PREREQUISITES]
+    prerequisites = dict(PRIVATE_PREREQUISITES)
 
-    assert CURRENT_LCFS_RELEASE.raw_zip_name in prerequisite_names
-    assert "lcfs_2021_22.zip" not in prerequisite_names
+    assert prerequisites[CURRENT_LCFS_RELEASE.raw_zip_name] == (
+        CURRENT_LCFS_RELEASE.ukds_tab_subdir
+    )
+    assert "lcfs_2021_22.zip" not in prerequisites
 
 
 def test_private_prerequisites_use_current_was_release():
-    prerequisite_names = [filename for filename, _ in PRIVATE_PREREQUISITES]
+    prerequisites = dict(PRIVATE_PREREQUISITES)
 
-    assert CURRENT_WAS_RELEASE.raw_zip_name in prerequisite_names
-    assert "was_2006_20.zip" not in prerequisite_names
+    assert prerequisites[CURRENT_WAS_RELEASE.raw_zip_name] == (
+        CURRENT_WAS_RELEASE.ukds_tab_subdir
+    )
+    assert "was_2006_20.zip" not in prerequisites
 
 
 def test_private_prerequisites_use_current_etb_release():
-    prerequisite_names = [filename for filename, _ in PRIVATE_PREREQUISITES]
+    prerequisites = dict(PRIVATE_PREREQUISITES)
 
-    assert CURRENT_ETB_RELEASE.raw_zip_name in prerequisite_names
-    assert "etb_1977_21.zip" not in prerequisite_names
+    assert prerequisites[CURRENT_ETB_RELEASE.raw_zip_name] == (
+        CURRENT_ETB_RELEASE.ukds_tab_subdir
+    )
+    assert "etb_1977_21.zip" not in prerequisites
 
 
 def test_current_frs_release_uses_survey_year_as_base_year():

policyengine_uk_data/tests/test_private_releases.py

@@ -9,10 +9,9 @@ def test_current_lcfs_release_points_to_2023_24_ukds_files():
     assert CURRENT_LCFS_RELEASE.name == "lcfs_2023_24"
     assert CURRENT_LCFS_RELEASE.ukds_study_number == 9468
     assert CURRENT_LCFS_RELEASE.doi == "10.5255/UKDA-SN-9468-3"
-    assert CURRENT_LCFS_RELEASE.household_tab_filename == "9468_dvhh_ukanon_v2_2023.tab"
-    assert (
-        CURRENT_LCFS_RELEASE.person_tab_filename == "9468_dvper_ukanon_202324_2023.tab"
-    )
+    assert CURRENT_LCFS_RELEASE.ukds_tab_subdir == "UKDA-9468-tab/tab"
+    assert CURRENT_LCFS_RELEASE.household_tab_filename == "dvhh_ukanon_v2_2023.tab"
+    assert CURRENT_LCFS_RELEASE.person_tab_filename == "dvper_ukanon_202324_2023.tab"
     assert CURRENT_LCFS_RELEASE.fuel_price_year == 2023
 
 
@@ -21,9 +20,14 @@ def test_current_was_release_points_to_round_8_ukds_files():
     assert CURRENT_WAS_RELEASE.latest_round == 8
     assert CURRENT_WAS_RELEASE.ukds_study_number == 7215
     assert CURRENT_WAS_RELEASE.doi == "10.5255/UKDA-SN-7215-20"
+    assert CURRENT_WAS_RELEASE.ukds_tab_subdir == "UKDA-7215-tab/tab"
     assert (
         CURRENT_WAS_RELEASE.household_tab_filename
-        == "7215_was_round_8_hhold_eul_may_2025_230525.tab"
+        == "was_round_8_hhold_eul_may_2025_230525.tab"
+    )
+    assert (
+        CURRENT_WAS_RELEASE.person_tab_filename
+        == "was_round_8_person_eul_may_2025_230525.tab"
     )
 
 
@@ -33,9 +37,9 @@ def test_current_etb_release_points_to_2023_24_ukds_files():
     assert CURRENT_ETB_RELEASE.default_training_year == 2023
     assert CURRENT_ETB_RELEASE.ukds_study_number == 8856
     assert CURRENT_ETB_RELEASE.doi == "10.5255/UKDA-SN-8856-4"
-    assert (
-        CURRENT_ETB_RELEASE.household_tab_filename == "8856_householdv2_1977-2024.tab"
-    )
+    assert CURRENT_ETB_RELEASE.ukds_tab_subdir == "UKDA-8856-tab/tab"
+    assert CURRENT_ETB_RELEASE.household_tab_filename == "householdv2_1977-2024.tab"
+    assert CURRENT_ETB_RELEASE.person_tab_filename == "personv2_2018-2024.tab"
 
 
 def test_consumption_model_metadata_tracks_private_releases():