Fix v4 migrator: route annual techs' split rows to the annual split tables

temoa/utilities/master_migration.py

@@ -118,6 +118,29 @@ def get_table_info(conn: sqlite3.Connection, table: str) -> list[tuple[Any, ...]
         return []
 
 
+# v3-style split tables carry BOTH seasonal- and annual-tech rows; the v4 target depends
+# on the tech's `annual` flag. Temoa's seasonal split constraints skip annual techs, so an
+# annual tech's split row migrated into the seasonal table is silently dropped from the model.
+ANNUAL_SPLIT_TABLES = {
+    'TechInputSplit': 'limit_tech_input_split_annual',
+    'TechOutputSplit': 'limit_tech_output_split_annual',
+}
+
+
+def _get_annual_techs(con_old: sqlite3.Connection) -> set[str]:
+    """Techs flagged annual in the source DB (`Technology.annual`, with a `tech_annual`
+    table fallback for older v3 databases). Empty set if neither exists."""
+    for query in (
+        'SELECT tech FROM Technology WHERE annual = 1',
+        'SELECT tech FROM tech_annual',
+    ):
+        try:
+            return {r[0] for r in con_old.execute(query).fetchall()}
+        except sqlite3.OperationalError:
+            continue
+    return set()
+
+
 def _migrate_operator_tables(con_old: sqlite3.Connection, con_new: sqlite3.Connection) -> int:
     """Migrate max/min tables to operator constraints."""
     print('--- Migrating max/min tables to operator constraints ---')
@@ -156,6 +179,25 @@ def _migrate_operator_tables(con_old: sqlite3.Connection, con_new: sqlite3.Conne
                 ]
 
         placeholders = ','.join(['?'] * len(data[0]))
+
+        if old_name in ANNUAL_SPLIT_TABLES:
+            # Route by the tech's `annual` flag (see ANNUAL_SPLIT_TABLES note). The
+            # seasonal and annual targets are column-identical, so the transformed
+            # rows insert into either.
+            annual_techs = _get_annual_techs(con_old)
+            tech_index = new_cols.index('tech')
+            for target, rows in (
+                (new_name, [r for r in data if r[tech_index] not in annual_techs]),
+                (ANNUAL_SPLIT_TABLES[old_name], [r for r in data if r[tech_index] in annual_techs]),
+            ):
+                if rows:
+                    con_new.executemany(
+                        f'INSERT OR REPLACE INTO {target} VALUES ({placeholders})', rows
+                    )
+                    print(f'Migrated {len(rows)} rows: {old_name} -> {target}')
+            total += len(data)
+            continue
+
         query = f'INSERT OR REPLACE INTO {new_name} VALUES ({placeholders})'
         con_new.executemany(query, data)
         print(f'Migrated {len(data)} rows: {old_name} -> {new_name}')

tests/test_v4_migration.py

@@ -90,6 +90,24 @@ def test_v4_migrations(tmp_path: Path, schema_file: Path, mock_data_file: Path)
         _verify_migrated_data(conn_db)
 
 
+def test_get_annual_techs_source_variants() -> None:
+    """_get_annual_techs must read Technology.annual, fall back to a tech_annual
+    table (older v3 databases), and return an empty set when neither exists."""
+    from temoa.utilities.master_migration import _get_annual_techs
+
+    con = sqlite3.connect(':memory:')
+    con.execute('CREATE TABLE Technology (tech TEXT, annual INTEGER)')
+    con.executemany('INSERT INTO Technology VALUES (?, ?)', [('T1', 0), ('T2', 1)])
+    assert _get_annual_techs(con) == {'T2'}
+
+    con_old = sqlite3.connect(':memory:')
+    con_old.execute('CREATE TABLE tech_annual (tech TEXT)')
+    con_old.execute("INSERT INTO tech_annual VALUES ('T9')")
+    assert _get_annual_techs(con_old) == {'T9'}
+
+    assert _get_annual_techs(sqlite3.connect(':memory:')) == set()
+
+
 def _verify_migrated_data(conn: sqlite3.Connection) -> None:
     # Check time_season restructuring (aggregated from TimeSegmentFraction)
     # Summer: 0.4 + 0.3 = 0.7
@@ -125,6 +143,29 @@ def _verify_migrated_data(conn: sqlite3.Connection) -> None:
     assert len(cap_rows) == 1
     assert cap_rows[0] == ('R1', 'T1', 2030, 10.0, 'ge')
 
+    # Split-table routing: an annual tech's split must land in the *_annual table —
+    # the seasonal split constraints silently skip annual techs, so a mis-routed row
+    # disappears from the model without error.
+    in_seasonal = conn.execute(
+        'SELECT region, period, input_comm, tech, operator, proportion FROM limit_tech_input_split'
+    ).fetchall()
+    assert in_seasonal == [('R1', 2030, 'In', 'T1', 'ge', pytest.approx(0.3))]
+    in_annual = conn.execute(
+        'SELECT region, period, input_comm, tech, operator, proportion '
+        'FROM limit_tech_input_split_annual'
+    ).fetchall()
+    assert in_annual == [('R1', 2030, 'In', 'T2', 'ge', pytest.approx(0.4))]
+    out_seasonal = conn.execute(
+        'SELECT region, period, tech, output_comm, operator, proportion '
+        'FROM limit_tech_output_split'
+    ).fetchall()
+    assert out_seasonal == [('R1', 2030, 'T1', 'Out', 'ge', pytest.approx(0.5))]
+    out_annual = conn.execute(
+        'SELECT region, period, tech, output_comm, operator, proportion '
+        'FROM limit_tech_output_split_annual'
+    ).fetchall()
+    assert out_annual == [('R1', 2030, 'T2', 'Out', 'ge', pytest.approx(0.6))]
+
     emis_rows = conn.execute(
         'SELECT region, period, value, operator FROM limit_emission'
     ).fetchall()

tests/testing_data/migration_v3_1_mock.sql

@@ -30,3 +30,11 @@ INSERT INTO CapacityFactorProcess (region, period, season, tod, tech, vintage, f
 INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T1', 2030, 'Out', 0.9);
 INSERT INTO LimitCapacity (region, period, tech_or_group, operator, capacity, units, notes) VALUES ('R1', 2030, 'T1', 'ge', 10.0, 'GW', 'test op');
 INSERT INTO LimitEmission (region, period, emis_comm, operator, value, units, notes) VALUES ('R1', 2030, 'Out', 'le', 100.0, 'kt', 'test op');
+
+-- Split tables (explicit seasonal/annual in v3.1) — verify placement survives migration.
+INSERT INTO Technology (tech, flag, unlim_cap, annual, reserve, curtail, retire, flex, exchange, seas_stor) VALUES ('T2', 'p', 0, 1, 0, 0, 0, 0, 0, 0);
+INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T2', 2030, 'Out', 0.9);
+INSERT INTO LimitTechInputSplit (region, period, input_comm, tech, operator, proportion, notes) VALUES ('R1', 2030, 'In', 'T1', 'ge', 0.3, 'seasonal-tech split');
+INSERT INTO LimitTechInputSplitAnnual (region, period, input_comm, tech, operator, proportion, notes) VALUES ('R1', 2030, 'In', 'T2', 'ge', 0.4, 'annual-tech split');
+INSERT INTO LimitTechOutputSplit (region, period, tech, output_comm, operator, proportion, notes) VALUES ('R1', 2030, 'T1', 'Out', 'ge', 0.5, 'seasonal-tech split');
+INSERT INTO LimitTechOutputSplitAnnual (region, period, tech, output_comm, operator, proportion, notes) VALUES ('R1', 2030, 'T2', 'Out', 'ge', 0.6, 'annual-tech split');

tests/testing_data/migration_v3_mock.sql

@@ -30,3 +30,11 @@ INSERT INTO CapacityFactorProcess (region, season, tod, tech, vintage, factor) V
 INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T1', 2030, 'Out', 0.9);
 INSERT INTO MinCapacity (region, tech, period, min_cap, units, notes) VALUES ('R1', 'T1', 2030, 10.0, 'GW', 'test op');
 INSERT INTO EmissionLimit (region, period, emis_comm, value, units, notes) VALUES ('R1', 2030, 'Out', 100.0, 'kt', 'test op');
+
+-- Split-table routing: T2 is an annual tech; its split rows must land in the *_annual tables.
+INSERT INTO Technology (tech, flag, unlim_cap, annual, reserve, curtail, retire, flex, exchange) VALUES ('T2', 'p', 0, 1, 0, 0, 0, 0, 0);
+INSERT INTO Efficiency (region, input_comm, tech, vintage, output_comm, efficiency) VALUES ('R1', 'In', 'T2', 2030, 'Out', 0.9);
+INSERT INTO TechInputSplit (region, period, input_comm, tech, min_proportion, notes) VALUES ('R1', 2030, 'In', 'T1', 0.3, 'seasonal-tech split');
+INSERT INTO TechInputSplit (region, period, input_comm, tech, min_proportion, notes) VALUES ('R1', 2030, 'In', 'T2', 0.4, 'annual-tech split');
+INSERT INTO TechOutputSplit (region, period, tech, output_comm, min_proportion, notes) VALUES ('R1', 2030, 'T1', 'Out', 0.5, 'seasonal-tech split');
+INSERT INTO TechOutputSplit (region, period, tech, output_comm, min_proportion, notes) VALUES ('R1', 2030, 'T2', 'Out', 0.6, 'annual-tech split');