From 71a970c66fdd027e634cdf6c1f0734d85710a4e0 Mon Sep 17 00:00:00 2001 From: Cameron Wade Date: Tue, 14 Jul 2026 10:37:00 -0300 Subject: [PATCH 1/2] Route v3 split-table rows to the annual v4 tables for annual techs The v3->v4 migrator routed TechInputSplit and TechOutputSplit rows to the seasonal v4 tables (limit_tech_input_split / limit_tech_output_split) unconditionally. Temoa's seasonal split constraints only apply to non-annual techs, so every annual tech's split constraint silently disappeared from migrated models: the rows exist in the DB but no constraint is built from them. Fix: partition split rows by the tech's Technology.annual flag (with a tech_annual table fallback for older v3 databases) and route annual techs' rows to limit_tech_input_split_annual / limit_tech_output_split_annual. The seasonal and annual targets are column-identical, so the existing row transformation applies to both. Found migrating the Open Energy Outlook 9-region US database: 13,313 of 15,369 TechInputSplit rows belonged to annual techs (transport fuel blends, industrial fuel mixes) and were dropped from the model, letting e.g. Fischer-Tropsch synfuel run without its mandated hydrogen input. Adds regression coverage: both mock datasets now carry a seasonal-tech and an annual-tech split row, asserting placement after migration. The new v3-schema assertions fail without this fix. --- temoa/utilities/master_migration.py | 42 ++++++++++++++++++++++ tests/test_v4_migration.py | 23 ++++++++++++ tests/testing_data/migration_v3_1_mock.sql | 8 +++++ tests/testing_data/migration_v3_mock.sql | 8 +++++ 4 files changed, 81 insertions(+) diff --git a/temoa/utilities/master_migration.py b/temoa/utilities/master_migration.py index 6fccac448..69d0e5f26 100644 --- a/temoa/utilities/master_migration.py +++ b/temoa/utilities/master_migration.py @@ -118,6 +118,29 @@ def get_table_info(conn: sqlite3.Connection, table: str) -> list[tuple[Any, ...] return [] +# v3-style split tables carry BOTH seasonal- and annual-tech rows; the v4 target depends +# on the tech's `annual` flag. Temoa's seasonal split constraints skip annual techs, so an +# annual tech's split row migrated into the seasonal table is silently dropped from the model. +ANNUAL_SPLIT_TABLES = { + 'TechInputSplit': 'limit_tech_input_split_annual', + 'TechOutputSplit': 'limit_tech_output_split_annual', +} + + +def _get_annual_techs(con_old: sqlite3.Connection) -> set[str]: + """Techs flagged annual in the source DB (`Technology.annual`, with a `tech_annual` + table fallback for older v3 databases). Empty set if neither exists.""" + for query in ( + 'SELECT tech FROM Technology WHERE annual = 1', + 'SELECT tech FROM tech_annual', + ): + try: + return {r[0] for r in con_old.execute(query).fetchall()} + except sqlite3.OperationalError: + continue + return set() + + def _migrate_operator_tables(con_old: sqlite3.Connection, con_new: sqlite3.Connection) -> int: """Migrate max/min tables to operator constraints.""" print('--- Migrating max/min tables to operator constraints ---') @@ -156,6 +179,25 @@ def _migrate_operator_tables(con_old: sqlite3.Connection, con_new: sqlite3.Conne ] placeholders = ','.join(['?'] * len(data[0])) + + if old_name in ANNUAL_SPLIT_TABLES: + # Route by the tech's `annual` flag (see ANNUAL_SPLIT_TABLES note). The + # seasonal and annual targets are column-identical, so the transformed + # rows insert into either. + annual_techs = _get_annual_techs(con_old) + tech_index = new_cols.index('tech') + for target, rows in ( + (new_name, [r for r in data if r[tech_index] not in annual_techs]), + (ANNUAL_SPLIT_TABLES[old_name], [r for r in data if r[tech_index] in annual_techs]), + ): + if rows: + con_new.executemany( + f'INSERT OR REPLACE INTO {target} VALUES ({placeholders})', rows + ) + print(f'Migrated {len(rows)} rows: {old_name} -> {target}') + total += len(data) + continue + query = f'INSERT OR REPLACE INTO {new_name} VALUES ({placeholders})' con_new.executemany(query, data) print(f'Migrated {len(data)} rows: {old_name} -> {new_name}') diff --git a/tests/test_v4_migration.py b/tests/test_v4_migration.py index 0cd82fa1b..714916b82 100644 --- a/tests/test_v4_migration.py +++ b/tests/test_v4_migration.py @@ -125,6 +125,29 @@ def _verify_migrated_data(conn: sqlite3.Connection) -> None: assert len(cap_rows) == 1 assert cap_rows[0] == ('R1', 'T1', 2030, 10.0, 'ge') + # Split-table routing: an annual tech's split must land in the *_annual table — + # the seasonal split constraints silently skip annual techs, so a mis-routed row + # disappears from the model without error. + in_seasonal = conn.execute( + 'SELECT region, period, input_comm, tech, operator, proportion FROM limit_tech_input_split' + ).fetchall() + assert in_seasonal == [('R1', 2030, 'In', 'T1', 'ge', pytest.approx(0.3))] + in_annual = conn.execute( + 'SELECT region, period, input_comm, tech, operator, proportion ' + 'FROM limit_tech_input_split_annual' + ).fetchall() + assert in_annual == [('R1', 2030, 'In', 'T2', 'ge', pytest.approx(0.4))] + out_seasonal = conn.execute( + 'SELECT region, period, tech, output_comm, operator, proportion ' + 'FROM limit_tech_output_split' + ).fetchall() + assert out_seasonal == [('R1', 2030, 'T1', 'Out', 'ge', pytest.approx(0.5))] + out_annual = conn.execute( + 'SELECT region, period, tech, output_comm, operator, proportion ' + 'FROM limit_tech_output_split_annual' + ).fetchall() + assert out_annual == [('R1', 2030, 'T2', 'Out', 'ge', pytest.approx(0.6))] + emis_rows = conn.execute( 'SELECT region, period, value, operator FROM limit_emission' ).fetchall() diff --git a/tests/testing_data/migration_v3_1_mock.sql b/tests/testing_data/migration_v3_1_mock.sql index 9a282962f..94d8cb69d 100644 --- a/tests/testing_data/migration_v3_1_mock.sql +++ b/tests/testing_data/migration_v3_1_mock.sql @@ -30,3 +30,11 @@ INSERT INTO CapacityFactorProcess (region, period, season, tod, tech, vintage, f INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T1', 2030, 'Out', 0.9); INSERT INTO LimitCapacity (region, period, tech_or_group, operator, capacity, units, notes) VALUES ('R1', 2030, 'T1', 'ge', 10.0, 'GW', 'test op'); INSERT INTO LimitEmission (region, period, emis_comm, operator, value, units, notes) VALUES ('R1', 2030, 'Out', 'le', 100.0, 'kt', 'test op'); + +-- Split tables (explicit seasonal/annual in v3.1) — verify placement survives migration. +INSERT INTO Technology (tech, flag, unlim_cap, annual, reserve, curtail, retire, flex, exchange, seas_stor) VALUES ('T2', 'p', 0, 1, 0, 0, 0, 0, 0, 0); +INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T2', 2030, 'Out', 0.9); +INSERT INTO LimitTechInputSplit (region, period, input_comm, tech, operator, proportion, notes) VALUES ('R1', 2030, 'In', 'T1', 'ge', 0.3, 'seasonal-tech split'); +INSERT INTO LimitTechInputSplitAnnual (region, period, input_comm, tech, operator, proportion, notes) VALUES ('R1', 2030, 'In', 'T2', 'ge', 0.4, 'annual-tech split'); +INSERT INTO LimitTechOutputSplit (region, period, tech, output_comm, operator, proportion, notes) VALUES ('R1', 2030, 'T1', 'Out', 'ge', 0.5, 'seasonal-tech split'); +INSERT INTO LimitTechOutputSplitAnnual (region, period, tech, output_comm, operator, proportion, notes) VALUES ('R1', 2030, 'T2', 'Out', 'ge', 0.6, 'annual-tech split'); diff --git a/tests/testing_data/migration_v3_mock.sql b/tests/testing_data/migration_v3_mock.sql index 28973eb24..32d799453 100644 --- a/tests/testing_data/migration_v3_mock.sql +++ b/tests/testing_data/migration_v3_mock.sql @@ -30,3 +30,11 @@ INSERT INTO CapacityFactorProcess (region, season, tod, tech, vintage, factor) V INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T1', 2030, 'Out', 0.9); INSERT INTO MinCapacity (region, tech, period, min_cap, units, notes) VALUES ('R1', 'T1', 2030, 10.0, 'GW', 'test op'); INSERT INTO EmissionLimit (region, period, emis_comm, value, units, notes) VALUES ('R1', 2030, 'Out', 100.0, 'kt', 'test op'); + +-- Split-table routing: T2 is an annual tech; its split rows must land in the *_annual tables. +INSERT INTO Technology (tech, flag, unlim_cap, annual, reserve, curtail, retire, flex, exchange) VALUES ('T2', 'p', 0, 1, 0, 0, 0, 0, 0); +INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T2', 2030, 'Out', 0.9); +INSERT INTO TechInputSplit (region, period, input_comm, tech, min_proportion, notes) VALUES ('R1', 2030, 'In', 'T1', 0.3, 'seasonal-tech split'); +INSERT INTO TechInputSplit (region, period, input_comm, tech, min_proportion, notes) VALUES ('R1', 2030, 'In', 'T2', 0.4, 'annual-tech split'); +INSERT INTO TechOutputSplit (region, period, tech, output_comm, min_proportion, notes) VALUES ('R1', 2030, 'T1', 'Out', 0.5, 'seasonal-tech split'); +INSERT INTO TechOutputSplit (region, period, tech, output_comm, min_proportion, notes) VALUES ('R1', 2030, 'T2', 'Out', 0.6, 'annual-tech split'); From 18e287cbe1b56642786e3d7316c5b673ff7be42d Mon Sep 17 00:00:00 2001 From: Cameron Wade Date: Wed, 15 Jul 2026 09:28:19 -0300 Subject: [PATCH 2/2] Cover _get_annual_techs source variants (Technology.annual, tech_annual fallback, neither) --- tests/test_v4_migration.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_v4_migration.py b/tests/test_v4_migration.py index 714916b82..7c105aeb7 100644 --- a/tests/test_v4_migration.py +++ b/tests/test_v4_migration.py @@ -90,6 +90,24 @@ def test_v4_migrations(tmp_path: Path, schema_file: Path, mock_data_file: Path) _verify_migrated_data(conn_db) +def test_get_annual_techs_source_variants() -> None: + """_get_annual_techs must read Technology.annual, fall back to a tech_annual + table (older v3 databases), and return an empty set when neither exists.""" + from temoa.utilities.master_migration import _get_annual_techs + + con = sqlite3.connect(':memory:') + con.execute('CREATE TABLE Technology (tech TEXT, annual INTEGER)') + con.executemany('INSERT INTO Technology VALUES (?, ?)', [('T1', 0), ('T2', 1)]) + assert _get_annual_techs(con) == {'T2'} + + con_old = sqlite3.connect(':memory:') + con_old.execute('CREATE TABLE tech_annual (tech TEXT)') + con_old.execute("INSERT INTO tech_annual VALUES ('T9')") + assert _get_annual_techs(con_old) == {'T9'} + + assert _get_annual_techs(sqlite3.connect(':memory:')) == set() + + def _verify_migrated_data(conn: sqlite3.Connection) -> None: # Check time_season restructuring (aggregated from TimeSegmentFraction) # Summer: 0.4 + 0.3 = 0.7