Update test targets

pyproject.toml

@@ -8,10 +8,10 @@ dependencies = [
     "activitysim>=1.5",
     "multimethod<2.0",
     "numba>=0.57",
-    "numpy>=1.16.1,<1.26",
+    "numpy>=2,<3",
     "openmatrix>=0.3.5.0",
     "pandas>=2.1.0",
-    "pandera>=0.15,<0.18.1",
+    "pandera>=0.30",
     "psutil>=4.1",
     "pydantic>=2.6",
     "setuptools>=80.9.0,<81",
@@ -27,4 +27,4 @@ dev = [
 
 [tool.uv.sources]
 activitysim = { git = "https://github.com/ActivitySim/ActivitySim", branch = "main" }
-# TODO: unpin from source once we have a release with the new features we need
+## TODO: unpin from source once we have a release with the new features we need

scripts/fulldata.py

@@ -96,3 +96,9 @@ def get_full_data(full_data_dir):
     # write a gitignore to the full data dir to avoid accidental commits
     if not full_data_dir.joinpath(".gitignore").exists():
         full_data_dir.joinpath(".gitignore").write_text("**\n")
+
+
+if __name__ == "__main__":
+    import pathlib
+
+    get_full_data(pathlib.Path("/tmp/sandag-abm3-example"))

test/regress/final_vehicles.csv

@@ -1,104 +1,102 @@
 "household_id","vehicle_num","vehicle_type","auto_operating_cost","Range","MPG","vehicle_year","vehicle_id"
-784728,1,"Car_3_Gas",29.044636256912945,0,30.339852997086037,2020,7847281
-786196,1,"SUV_11_Gas",35.22952917036238,0,19.5,2012,7861961
-786980,1,"Pickup_20_Gas",40.83992347100951,0,15.8,2003,7869801
-788147,1,"Car_12_Gas",32.13524424275512,0,21.7,2011,7881471
-789606,1,"Car_20_Gas",33.34510499891135,0,20.5,2003,7896061
-791851,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,7918511
-791851,2,"Car_9_Diesel",25.70505837954799,0,31.5,2014,7918512
-791940,1,"SUV_10_Gas",34.654522172345345,0,20,2013,7919401
-796337,1,"SUV_11_Gas",35.22952917036238,0,19.5,2012,7963371
-796974,1,"Car_5_Hybrid",22.998974553004228,0,45.88043377690312,2018,7969741
-796974,2,"Pickup_20_Gas",40.83992347100951,0,15.8,2003,7969742
-797235,1,"Car_8_Gas",30.63368985519484,0,23.4,2015,7972351
-799627,1,"Car_20_Gas",33.34510499891135,0,20.5,2003,7996271
-803225,1,"Car_19_Gas",33.238899592783525,0,20.6,2004,8032251
-805047,1,"Car_13_Gas",32.32750893708441,0,21.5,2010,8050471
-805047,2,"Car_10_Gas",30.882610641776562,0,23.1,2013,8050472
-807979,1,"Car_7_Gas",30.391070869383654,0,23.7,2016,8079791
-808447,1,"Car_14_Gas",32.824192738243504,0,21,2009,8084471
-812300,1,"Pickup_10_Gas",38.99228655491246,0,16.9,2013,8123001
-812688,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,8126881
-812688,2,"Pickup_19_Gas",39.9691743472184,0,16.3,2004,8126882
-816340,1,"Car_11_Gas",31.48935503757862,0,22.4,2012,8163401
-816592,1,"Pickup_7_Gas",36.697040303846975,0,18.5,2016,8165921
-818181,1,"Car_11_Gas",31.48935503757862,0,22.4,2012,8181811
-818678,1,"Car_3_Gas",29.044636256912945,0,30.339852997086037,2020,8186781
-818816,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,8188161
-820094,1,"Car_12_Gas",32.13524424275512,0,21.7,2011,8200941
-822459,1,"SUV_7_Gas",33.18744822703672,0,21.4,2016,8224591
-822459,2,"Pickup_14_Gas",39.1502555164903,0,16.8,2009,8224592
-823163,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,8231631
-823163,2,"Van_12_Gas",41.37844084189838,0,15.1,2011,8231632
-823520,1,"Car_4_Gas",29.361094499800235,0,29.891480785306445,2019,8235201
-823520,2,"Car_20_Gas",33.34510499891135,0,20.5,2003,8235202
-824076,1,"SUV_5_Gas",32.66249476661974,0,26.005885999114497,2018,8240761
-824360,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,8243601
-824360,2,"Pickup_4_Gas",36.098599584298725,0,23.09647750296356,2019,8243602
-824360,3,"SUV_5_Gas",32.66249476661974,0,26.005885999114497,2018,8243603
-824748,1,"Pickup_14_Gas",39.1502555164903,0,16.8,2009,8247481
-825301,1,"Car_12_Gas",32.13524424275512,0,21.7,2011,8253011
-830248,1,"Car_15_Gas",33.34510499891135,0,20.5,2008,8302481
-830248,2,"Van_9_Gas",40.243319230588035,0,15.7,2014,8302482
-830642,1,"Car_5_Gas",29.67755274268745,0,29.449734763848717,2018,8306421
-830642,2,"Car_2_Gas",28.728178014025737,0,30.794950792042325,2021,8306422
-833913,1,"Car_20_Gas",33.34510499891135,0,20.5,2003,8339131
-833913,2,"Car_16_Gas",33.56065490090841,0,20.3,2007,8339132
-833995,1,"Car_7_Gas",30.391070869383654,0,23.7,2016,8339951
-834673,1,"Car_12_Gas",32.13524424275512,0,21.7,2011,8346731
-836353,1,"Car_3_Hybrid",22.514413326292903,0,47.267169887810006,2020,8363531
-836353,2,"Car_5_Gas",29.67755274268745,0,29.449734763848717,2018,8363532
-837064,1,"Pickup_7_Gas",36.697040303846975,0,18.5,2016,8370641
-838789,1,"SUV_3_Gas",31.858247688675156,0,26.791913903437724,2020,8387891
-838789,2,"Car_10_Gas",30.882610641776562,0,23.1,2013,8387892
-841437,1,"Pickup_12_Gas",38.37867302882849,0,17.3,2011,8414371
-841609,1,"Van_1_Gas",34.90915577502541,0,22.54594266184607,2022,8416091
-845801,1,"Car_3_Gas",29.044636256912945,0,30.339852997086037,2020,8458011
-845801,2,"SUV_4_Gas",32.26037122764736,0,26.39597428910121,2019,8458012
-847093,1,"Car_6_Gas",30.15451735186392,0,24,2017,8470931
-847093,2,"Car_20_Gas",33.34510499891135,0,20.5,2003,8470932
-847266,1,"Car_3_Gas",29.044636256912945,0,30.339852997086037,2020,8472661
-849829,1,"Van_9_Gas",40.243319230588035,0,15.7,2014,8498291
-850412,1,"SUV_2_Gas",31.45612414970278,0,27.193792611989288,2021,8504121
-850412,2,"Van_10_Gas",39.883983051641096,0,15.9,2013,8504122
-851027,1,"SUV_12_Gas",35.34808732892535,0,19.4,2011,8510271
-852013,1,"SUV_7_Diesel",30.16946753399176,0,25,2016,8520131
-859711,1,"Car_1_Gas",28.41171977113853,0,31.25687505392296,2022,8597111
-859711,2,"Car_2_Gas",28.728178014025737,0,30.794950792042325,2021,8597112
-860567,1,"Van_2_Gas",35.388869428319055,0,22.21275139098135,2021,8605671
-860567,2,"Car_10_Gas",30.882610641776562,0,23.1,2013,8605672
-861865,1,"Car_1_Gas",28.41171977113853,0,31.25687505392296,2022,8618651
-861865,2,"Car_8_Gas",30.63368985519484,0,23.4,2015,8618652
-862635,1,"Car_15_Gas",33.34510499891135,0,20.5,2008,8626351
-863066,1,"SUV_2_Gas",31.45612414970278,0,27.193792611989288,2021,8630661
-863724,1,"Car_15_Gas",33.34510499891135,0,20.5,2008,8637241
-865747,1,"Pickup_14_Gas",39.1502555164903,0,16.8,2009,8657471
-866001,1,"Car_8_Gas",30.63368985519484,0,23.4,2015,8660011
-868110,1,"Car_19_Gas",33.238899592783525,0,20.6,2004,8681101
-868110,2,"Car_4_BEV",15.403056255769519,147.82398967596902,116.75645063025708,2019,8681102
-868143,1,"SUV_6_Gas",33.385167091660435,0,21.2,2017,8681431
-868143,2,"Car_1_Gas",28.41171977113853,0,31.25687505392296,2022,8681432
-868503,1,"Pickup_9_Gas",38.6818914117807,0,17.1,2014,8685031
-868503,2,"Car_3_Gas",29.044636256912945,0,30.339852997086037,2020,8685032
-868503,3,"SUV_20_Gas",38.92600278457262,0,16.8,2003,8685033
-868503,4,"Car_20_Gas",33.34510499891135,0,20.5,2003,8685034
-868536,1,"Van_11_Gas",40.79982365895093,0,15.4,2012,8685361
-868536,2,"Car_7_Gas",30.391070869383654,0,23.7,2016,8685362
-868536,3,"Car_10_Gas",30.882610641776562,0,23.1,2013,8685363
-868536,4,"Motorcycle_16_Gas",18.673341637751523,0,53,2007,8685364
-868615,1,"Car_20_Gas",33.34510499891135,0,20.5,2003,8686151
-868615,2,"Van_3_Gas",35.86858308161269,0,21.884484129045667,2020,8686152
-868944,1,"Car_1_Gas",28.41171977113853,0,31.25687505392296,2022,8689441
-871938,1,"Motorcycle_2_Gas",18.673341637751523,0,71.35004992254615,2021,8719381
-872090,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,8720901
-872090,2,"Car_9_Gas",30.798921749802023,0,23.2,2014,8720902
-872123,1,"SUV_9_Gas",34.32311418942852,0,20.3,2014,8721231
-872123,2,"Pickup_7_Gas",36.697040303846975,0,18.5,2016,8721232
-872550,1,"Car_5_Gas",29.67755274268745,0,29.449734763848717,2018,8725501
-872550,2,"SUV_10_Gas",34.654522172345345,0,20,2013,8725502
-872550,3,"SUV_13_Gas",35.95969679804527,0,18.9,2010,8725503
-872713,1,"Car_1_Gas",28.41171977113853,0,31.25687505392296,2022,8727131
-872821,1,"Car_9_Gas",30.798921749802023,0,23.2,2014,8728211
-872821,2,"Car_10_Gas",30.882610641776562,0,23.1,2013,8728212
-873254,1,"SUV_5_Gas",32.66249476661974,0,26.005885999114497,2018,8732541
-873254,2,"Van_20_Gas",40.06251400036156,0,15.8,2003,8732542
+784728,1,"Car_3_Gas",29.111220259180275,0,30.339852997086037,2020,7847281
+786196,1,"SUV_11_Gas",35.310291863666826,0,19.5,2012,7861961
+786980,1,"Pickup_20_Gas",40.93354783362634,0,15.8,2003,7869801
+788147,1,"Car_12_Gas",32.20891337589882,0,21.7,2011,7881471
+789606,1,"Car_20_Gas",33.42154770341668,0,20.5,2003,7896061
+791851,1,"Car_9_Gas",30.86952740165046,0,23.2,2014,7918511
+791851,2,"Car_9_Diesel",25.763986494546096,0,31.5,2014,7918512
+791940,1,"SUV_10_Gas",34.73396667846641,0,20,2013,7919401
+796337,1,"SUV_11_Gas",35.310291863666826,0,19.5,2012,7963371
+796974,1,"Car_5_Hybrid",23.051699047821025,0,45.88043377690312,2018,7969741
+796974,2,"Pickup_20_Gas",40.93354783362634,0,15.8,2003,7969742
+797235,1,"Car_8_Gas",30.70391671762592,0,23.4,2015,7972351
+799627,1,"Car_20_Gas",33.42154770341668,0,20.5,2003,7996271
+803225,1,"Car_19_Gas",33.31509882441696,0,20.6,2004,8032251
+805047,1,"Car_13_Gas",32.401618831569714,0,21.5,2010,8050471
+805047,2,"Car_10_Gas",30.953408148028807,0,23.1,2013,8050472
+807979,1,"Car_7_Gas",30.460741534692534,0,23.7,2016,8079791
+808447,1,"Car_14_Gas",32.89944126621177,0,21,2009,8084471
+812300,1,"Pickup_10_Gas",39.08167526246643,0,16.9,2013,8123001
+812688,1,"Car_9_Gas",30.86952740165046,0,23.2,2014,8126881
+812688,2,"Pickup_19_Gas",40.0608025422427,0,16.3,2004,8126882
+816340,1,"Car_11_Gas",31.561543488095698,0,22.4,2012,8163401
+816592,1,"Pickup_7_Gas",36.78116722467264,0,18.5,2016,8165921
+818181,1,"Car_11_Gas",31.561543488095698,0,22.4,2012,8181811
+818678,1,"Car_3_Gas",29.111220259180275,0,30.339852997086037,2020,8186781
+818816,1,"Car_9_Gas",30.86952740165046,0,23.2,2014,8188161
+820094,1,"Car_12_Gas",32.20891337589882,0,21.7,2011,8200941
+822459,1,"SUV_7_Gas",33.26352950787802,0,21.4,2016,8224591
+822459,2,"Pickup_14_Gas",39.24000636339429,0,16.8,2009,8224592
+823163,1,"Car_8_Gas",30.70391671762592,0,23.4,2015,8231631
+823520,1,"Car_4_Gas",29.428403973585798,0,29.891480785306445,2019,8235201
+823520,2,"Car_20_Gas",33.42154770341668,0,20.5,2003,8235202
+824076,1,"SUV_5_Gas",32.73737260658854,0,26.005885999114497,2018,8240761
+824360,1,"Car_9_Gas",30.86952740165046,0,23.2,2014,8243601
+824360,2,"Pickup_4_Gas",36.18135459680112,0,23.09647750296356,2019,8243602
+824360,3,"SUV_5_Gas",32.73737260658854,0,26.005885999114497,2018,8243603
+824748,1,"Pickup_14_Gas",39.24000636339429,0,16.8,2009,8247481
+825301,1,"Car_12_Gas",32.20891337589882,0,21.7,2011,8253011
+830248,1,"Car_15_Gas",33.42154770341668,0,20.5,2008,8302481
+830248,2,"Van_9_Gas",40.33557589495774,0,15.7,2014,8302482
+830642,1,"Car_5_Gas",29.745587687991247,0,29.449734763848717,2018,8306421
+830642,2,"Car_2_Gas",28.79403654477483,0,30.794950792042325,2021,8306422
+833913,1,"Car_20_Gas",33.42154770341668,0,20.5,2003,8339131
+833913,2,"Car_16_Gas",33.63759174743143,0,20.3,2007,8339132
+833995,1,"Car_7_Gas",30.460741534692534,0,23.7,2016,8339951
+834673,1,"Car_12_Gas",32.20891337589882,0,21.7,2011,8346731
+836353,1,"Car_3_Hybrid",22.566026978283762,0,47.267169887810006,2020,8363531
+836353,2,"Car_5_Gas",29.745587687991247,0,29.449734763848717,2018,8363532
+837064,1,"Pickup_7_Gas",36.78116722467264,0,18.5,2016,8370641
+838789,1,"SUV_3_Gas",31.931281815099478,0,26.791913903437724,2020,8387891
+838789,2,"Car_10_Gas",30.953408148028807,0,23.1,2013,8387892
+841437,1,"Pickup_12_Gas",38.46665504483188,0,17.3,2011,8414371
+841609,1,"Van_1_Gas",34.989184021436024,0,22.54594266184607,2022,8416091
+845801,1,"Car_3_Gas",29.111220259180275,0,30.339852997086037,2020,8458011
+845801,2,"SUV_4_Gas",32.334327210843924,0,26.39597428910121,2019,8458012
+847093,1,"Car_6_Gas",30.22364572496409,0,24,2017,8470931
+847093,2,"Car_20_Gas",33.42154770341668,0,20.5,2003,8470932
+847266,1,"Car_3_Gas",29.111220259180275,0,30.339852997086037,2020,8472661
+849829,1,"Van_9_Gas",40.33557589495774,0,15.7,2014,8498291
+850412,1,"SUV_2_Gas",31.52823641935486,0,27.193792611989288,2021,8504121
+850412,2,"Van_10_Gas",39.97541594804407,0,15.9,2013,8504122
+851027,1,"SUV_12_Gas",35.429121813435195,0,19.4,2011,8510271
+852013,1,"SUV_7_Diesel",30.23863017995911,0,25,2016,8520131
+859711,1,"Car_1_Gas",28.476852830369385,0,31.25687505392296,2022,8597111
+859711,2,"Car_2_Gas",28.79403654477483,0,30.794950792042325,2021,8597112
+860567,1,"Van_2_Gas",35.469997404631464,0,22.21275139098135,2021,8605671
+860567,2,"Car_10_Gas",30.953408148028807,0,23.1,2013,8605672
+861865,1,"Car_1_Gas",28.476852830369385,0,31.25687505392296,2022,8618651
+861865,2,"Car_8_Gas",30.70391671762592,0,23.4,2015,8618652
+862635,1,"Car_15_Gas",33.42154770341668,0,20.5,2008,8626351
+863066,1,"SUV_2_Gas",31.52823641935486,0,27.193792611989288,2021,8630661
+863724,1,"Car_15_Gas",33.42154770341668,0,20.5,2008,8637241
+865747,1,"Pickup_14_Gas",39.24000636339429,0,16.8,2009,8657471
+866001,1,"Car_8_Gas",30.70391671762592,0,23.4,2015,8660011
+868110,1,"Car_19_Gas",33.31509882441696,0,20.6,2004,8681101
+868110,2,"Car_4_BEV",15.438367324002085,147.82398967596902,116.75645063025708,2019,8681102
+868143,1,"SUV_6_Gas",33.46170163737357,0,21.2,2017,8681431
+868143,2,"Car_1_Gas",28.476852830369385,0,31.25687505392296,2022,8681432
+868503,1,"Pickup_9_Gas",38.770568547300115,0,17.1,2014,8685031
+868503,2,"Car_3_Gas",29.111220259180275,0,30.339852997086037,2020,8685032
+868503,3,"SUV_20_Gas",39.015239538468926,0,16.8,2003,8685033
+868503,4,"Car_20_Gas",33.42154770341668,0,20.5,2003,8685034
+868536,1,"Van_11_Gas",40.893356093889494,0,15.4,2012,8685361
+868536,2,"Car_7_Gas",30.460741534692534,0,23.7,2016,8685362
+868536,3,"Car_10_Gas",30.953408148028807,0,23.1,2013,8685363
+868536,4,"Motorcycle_16_Gas",18.716149742179088,0,53,2007,8685364
+868615,1,"Car_20_Gas",33.42154770341668,0,20.5,2003,8686151
+868615,2,"Van_3_Gas",35.950810787826896,0,21.884484129045667,2020,8686152
+868944,1,"Car_1_Gas",28.476852830369385,0,31.25687505392296,2022,8689441
+871938,1,"Motorcycle_2_Gas",18.716149742179088,0,71.35004992254615,2021,8719381
+872090,1,"Car_8_Gas",30.70391671762592,0,23.4,2015,8720901
+872123,1,"SUV_9_Gas",34.40179895217767,0,20.3,2014,8721231
+872123,2,"Pickup_7_Gas",36.78116722467264,0,18.5,2016,8721232
+872550,1,"Car_5_Gas",29.745587687991247,0,29.449734763848717,2018,8725501
+872550,2,"SUV_10_Gas",34.73396667846641,0,20,2013,8725502
+872550,3,"SUV_13_Gas",36.042133379862115,0,18.9,2010,8725503
+872713,1,"Car_1_Gas",28.476852830369385,0,31.25687505392296,2022,8727131
+872821,1,"Car_9_Gas",30.86952740165046,0,23.2,2014,8728211
+872821,2,"Car_10_Gas",30.953408148028807,0,23.1,2013,8728212
+873254,1,"SUV_5_Gas",32.73737260658854,0,26.005885999114497,2018,8732541
+873254,2,"Van_20_Gas",40.15435617388508,0,15.8,2003,8732542

test/test_sandag_abm3.py

@@ -106,7 +106,6 @@ def regress(out_dir: Path, regress_dir: Path = None, filename="final_trips.csv")
     "write_tables",
 ]
 
-
 @pytest.mark.parametrize("use_sharrow", [False, True])
 def test_sandag_abm3_progressive(use_sharrow):
     import activitysim.abm  # register components # noqa: F401
@@ -157,7 +156,17 @@ def test_sandag_abm3_progressive(use_sharrow):
         if ref_pipeline.exists():
             try:
                 # The usual default rtol=1e-5 is too strict for cross-platform testing
-                state.checkpoint.check_against(ref_pipeline, checkpoint_name=step_name, rtol=1e-4)
+                try:
+                    state.checkpoint.check_against(ref_pipeline, checkpoint_name=step_name, rtol=1e-4)
+                except FileNotFoundError as err:
+                    missing_location = Path(str(err))
+                    computed_source = state.checkpoint.store.filename / missing_location.relative_to(missing_location.parents[1])
+                    if missing_location.parents[1].is_dir():
+                        missing_location.parents[0].mkdir(parents=True, exist_ok=True)
+                        shutil.copy(computed_source, missing_location)
+                        raise RuntimeError(f"> sandag-abm3 {step_name}: MISSING FILE REPLACED @ {missing_location}") from err
+                    raise err
+
             except Exception:
                 print(f"> sandag-abm3 {step_name}: ERROR")
                 raise
@@ -168,8 +177,6 @@ def test_sandag_abm3_progressive(use_sharrow):
 
     if not ref_pipeline.exists():
         # make new reference pipeline file if it is missing
-        import shutil
-
         if ref_pipeline.suffix == ".zip":
             shutil.make_archive(
                 ref_pipeline.with_suffix(""), "zip", state.checkpoint.store.filename