Fix treatment of early retirement in myopic mode (#340)

* [pre-commit.ci] pre-commit autoupdate

updates:
- [github.com/astral-sh/uv-pre-commit: 0.11.19 → 0.11.21](astral-sh/uv-pre-commit@0.11.19...0.11.21)
- [github.com/astral-sh/ruff-pre-commit: v0.15.16 → v0.15.17](astral-sh/ruff-pre-commit@v0.15.16...v0.15.17)

* [pre-commit.ci] pre-commit autoupdate

updates:
- [github.com/astral-sh/uv-pre-commit: 0.11.21 → 0.11.23](astral-sh/uv-pre-commit@0.11.21...0.11.23)
- [github.com/astral-sh/ruff-pre-commit: v0.15.17 → v0.15.18](astral-sh/ruff-pre-commit@v0.15.17...v0.15.18)

* Load early retirements from previous planning periods

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Adjust existing capacity for past early retirement in myopic

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Fix handling of existing capacity for p0

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Pull lifetimes data for previously retired existing capacities for accounting purposes

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Actually load growthrate constraints

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Add a broad stress test for myopic that includes survival curves, early retirement, and growth rate constraints all together

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Update test set hashes

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Improve custom loader filtering for lifetime data

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Update existing capacity check to specifically look for tiny dropped capacities

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Try to fix python 3.12 type error

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

* Check if output retirement table exists

Signed-off-by: Davey Elder <iandavidelder@gmail.com>

---------

Signed-off-by: Davey Elder <iandavidelder@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: idelder

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   - repo: https://github.com/astral-sh/uv-pre-commit
     # uv version.
-    rev: 0.11.19
+    rev: 0.11.23
     hooks:
       # Dependency management
       - id: uv-lock
@@ -47,7 +47,7 @@ repos:
     # Python Linting & Formatting with Ruff
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.15.16"
+    rev: "v0.15.18"
     hooks:
       - id: ruff
         name: ruff (linter)

temoa/components/capacity.py

@@ -18,7 +18,7 @@
 from deprecated import deprecated
 from pyomo.environ import value
 
-from .utils import get_capacity_factor
+from .utils import get_adjusted_existing_capacity, get_capacity_factor
 
 if TYPE_CHECKING:
     from temoa.core.model import TemoaModel
@@ -292,7 +292,9 @@ def annual_retirement_constraint(
         # Exact EOL. No v_capacity or v_retired_capacity for this period.
         if p == model.time_optimize.first():
             # Must be existing capacity. Apply survival curve to existing cap
-            cap_begin = model.existing_capacity[r, t, v] * model.lifetime_survival_curve[r, p, t, v]
+            cap_begin = get_adjusted_existing_capacity(model, r, t, v) * value(
+                model.lifetime_survival_curve[r, p, t, v]
+            )
         else:
             # Get previous capacity and continue survival curve
             p_prev = model.time_optimize.prev(p)
@@ -545,8 +547,9 @@ def adjusted_capacity_constraint(
     the time when that retirement occurred (treated here as at the beginning of each period).
     """
 
+    built_capacity: ExprLike
     if v in model.time_exist:
-        built_capacity = value(model.existing_capacity[r, t, v])
+        built_capacity = get_adjusted_existing_capacity(model, r, t, v)
     else:
         built_capacity = model.v_new_capacity[r, t, v]
 

temoa/components/limits.py

@@ -20,7 +20,12 @@
 
 import temoa.components.geography as geography
 import temoa.components.technology as technology
-from temoa.components.utils import Operator, get_variable_efficiency, operator_expression
+from temoa.components.utils import (
+    Operator,
+    get_adjusted_existing_capacity,
+    get_variable_efficiency,
+    operator_expression,
+)
 
 if TYPE_CHECKING:
     from pyomo.core import Expression
@@ -1043,11 +1048,10 @@ def limit_growth_capacity(
 
     if p == model.time_optimize.first():
         # First future period. Grab available capacity in last existing period
-        # Adjust in-line for past PLF because we are constraining available capacity
         p_prev = model.time_exist.last()
         capacity_prev = sum(
-            value(model.existing_capacity[_r, _t, _v])
-            * min(1.0, (_v + value(model.lifetime_process[_r, _t, _v]) - p_prev) / (p - p_prev))
+            get_adjusted_existing_capacity(model, _r, _t, _v)
+            * value(model.process_life_frac[_r, p_prev, _t, _v])
             for _r, _t, _v in model.existing_capacity.sparse_keys()
             if _r in regions
             and _t in techs

temoa/components/technology.py

@@ -17,6 +17,8 @@
 
 from pyomo.environ import value
 
+from temoa.components.utils import get_adjusted_existing_capacity
+
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
@@ -52,15 +54,31 @@ def model_process_life_indices(
     the periods in which a process is active, distinct from TechLifeFracIndices that
     returns indices only for processes that EOL mid-period.
     """
-    return model.active_activity_rptv
+    indices = {
+        (r, model.time_exist.last(), t, v) for r, t, v in model.existing_capacity.sparse_keys()
+    }
+    indices = indices | model.active_activity_rptv
+
+    return indices
+
+
+def lifetime_tech_indices(model: TemoaModel) -> set[tuple[Region, Technology]]:
+    """
+    Based on the efficiency parameter's indices, this function returns the set of
+    process indices that may be specified in the lifetime_tech parameter.
+    """
+    indices = {(r, t) for r, _, t, _, _ in set(model.efficiency.sparse_keys())}
+    indices = indices | {(r, t) for r, t, _ in model.existing_capacity.sparse_keys()}
+
+    return indices
 
 
 def lifetime_process_indices(model: TemoaModel) -> set[tuple[Region, Technology, Vintage]]:
     """
     Based on the efficiency parameter's indices, this function returns the set of
     process indices that may be specified in the lifetime_process parameter.
     """
-    indices = {(r, t, v) for r, i, t, v, o in set(model.efficiency.sparse_keys())}
+    indices = {(r, t, v) for r, _, t, v, _ in set(model.efficiency.sparse_keys())}
     indices = indices | set(model.existing_capacity.sparse_keys())
 
     return indices
@@ -424,14 +442,18 @@ def check_existing_capacity(model: TemoaModel) -> None:
             )
             logger.warning(msg)
             continue
-        if t not in model.tech_all:
+        adjusted_cap = get_adjusted_existing_capacity(model, r, t, v)
+        if adjusted_cap <= 0.0:
+            # Was retired in a previous period (myopic mode)
             continue
+        p = model.time_optimize.first()
         life = value(model.lifetime_process[r, t, v])
-        if (r, t, v) not in model.process_periods and v + life > model.time_optimize.first():
+        if (r, t, v) not in model.process_periods and v + life > p:
+            surviving_cap = adjusted_cap * value(model.lifetime_survival_curve[r, p, t, v])
             msg = (
-                f'Existing capacity {r, t, v} with lifetime {life} and capacity {cap} '
-                'should extend into future periods but it is not in process periods. '
-                'Was it included in the Efficiency table?'
+                f'Existing capacity {r, t, v} with lifetime {life} and surviving capacity '
+                f'{surviving_cap} should extend into future periods but is not an active '
+                'process. It may be missing from the Efficiency table or have too little '
+                'capacity to output and carry forward if running in myopic mode.'
             )
-            logger.error(msg)
-            raise ValueError(msg)
+            logger.warning(msg)

temoa/components/time.py

@@ -184,18 +184,18 @@ def init_set_time_optimize(model: TemoaModel) -> list[int]:
     return sorted(model.time_future)[:-1]
 
 
-def init_set_vintage_exist(model: TemoaModel) -> list[int]:
-    """Initializes the `vintage_exist` set."""
-    return sorted(model.time_exist)
-
-
 def init_set_vintage_optimize(model: TemoaModel) -> list[int]:
     """Initializes the `vintage_optimize` set."""
     return sorted(model.time_optimize)
 
 
 def param_period_length(model: TemoaModel, p: Period) -> int:
     """Rule to calculate the length of each optimization period in years."""
+    if model.time_exist and p == model.time_exist.last():
+        # Need this for one specific use case (capacity growth constraints)
+        return model.time_future.first() - model.time_exist.last()
+    elif p in model.time_exist:
+        return -1  # Period length is not defined for existing periods except the last
     periods: list[int] = sorted(model.time_future)
     i: int = periods.index(p)
     return periods[i + 1] - periods[i]

temoa/components/utils.py

@@ -87,3 +87,20 @@ def get_capacity_factor(
     if model.is_capacity_factor_process[r, t, v]:
         return value(model.capacity_factor_process[r, s, d, t, v])
     return value(model.capacity_factor_tech[r, s, d, t])
+
+
+def get_adjusted_existing_capacity(
+    model: TemoaModel, r: Region, t: Technology, v: Vintage
+) -> float:
+    """
+    Returns the built existing capacity adjusted for any early retirements.
+
+    Needed for early retirements in myopic mode. Takes into account survival curves
+    and any early retirements that may have occurred prior to this planning step.
+    """
+    capacity_adjustment = sum(
+        value(model.retired_existing_capacity[r, _p, t, v])
+        / (value(model.lifetime_survival_curve[r, _p, t, v]) or 1)
+        for _p in model.time_exist
+    )
+    return value(model.existing_capacity[r, t, v]) - capacity_adjustment

temoa/core/model.py

@@ -14,6 +14,7 @@
 from pyomo.core import BuildCheck, Set, Var
 from pyomo.environ import (
     AbstractModel,
+    Any,
     BuildAction,
     Constraint,
     Integers,
@@ -212,9 +213,9 @@ def __init__(self, *args: object, **kwargs: object) -> None:
             ordered=True, initialize=time.init_set_time_optimize, within=self.time_future
         )
         # Define time period vintages to track capacity installation
-        self.vintage_exist = Set(ordered=True, initialize=time.init_set_vintage_exist)
+        self.vintage_exist = Set(ordered=True)
         self.vintage_optimize = Set(ordered=True, initialize=time.init_set_vintage_optimize)
-        self.vintage_all = Set(initialize=self.time_exist | self.time_optimize)
+        self.vintage_all = Set(initialize=self.vintage_exist | self.time_optimize)
         # Perform some basic validation on the specified time periods.
         self.validate_time = BuildAction(rule=time.validate_time)
 
@@ -335,7 +336,9 @@ def __init__(self, *args: object, **kwargs: object) -> None:
         # Define time-related parameters
         # Basic period construction
         self.time_sequencing = Set()  # How do states carry between time segments?
-        self.period_length = Param(self.time_optimize, initialize=time.param_period_length)
+        self.period_length = Param(
+            self.time_optimize | self.time_exist, initialize=time.param_period_length
+        )
         self.days_per_period = Param(domain=PositiveReals, default=365.0)
         self.time_of_day_hours = Param(self.time_of_day, domain=PositiveReals, default=1.0)
         self.segment_fraction_per_season = Param(self.time_season)
@@ -383,6 +386,10 @@ def __init__(self, *args: object, **kwargs: object) -> None:
         self.capacity_to_activity = Param(self.regional_indices, self.tech_all, default=1)
 
         self.existing_capacity = Param(self.regional_indices, self.tech_exist, self.vintage_exist)
+        # This is needed to handle past retirements in myopic mode. Maybe it will find other uses.
+        self.retired_existing_capacity = Param(
+            self.regional_indices, self.time_exist, self.tech_exist, self.vintage_exist, default=0
+        )
 
         # Dev Note:  The below is temporarily useful for passing down to validator to find
         # set violations
@@ -431,9 +438,8 @@ def __init__(self, *args: object, **kwargs: object) -> None:
             default=1,
         )
 
-        self.lifetime_tech = Param(
-            self.regional_indices, self.tech_all, default=TemoaModel.default_lifetime_tech
-        )
+        self.lifetime_tech_rt = Set(dimen=2, initialize=technology.lifetime_tech_indices)
+        self.lifetime_tech = Param(self.lifetime_tech_rt, default=TemoaModel.default_lifetime_tech)
 
         self.lifetime_process_rtv = Set(dimen=3, initialize=technology.lifetime_process_indices)
         self.lifetime_process = Param(
@@ -443,7 +449,7 @@ def __init__(self, *args: object, **kwargs: object) -> None:
         self.lifetime_survival_curve = Param(
             self.regional_indices,
             Integers,
-            self.tech_all,
+            self.tech_all | self.tech_exist,
             self.vintage_all,
             default=technology.get_default_survival,
             validate=validate_0to1,
@@ -621,22 +627,22 @@ def __init__(self, *args: object, **kwargs: object) -> None:
         )
 
         self.limit_growth_capacity = Param(
-            self.regional_global_indices, self.tech_or_group, self.operator
+            self.regional_global_indices, self.tech_or_group, self.operator, within=Any
         )
         self.limit_degrowth_capacity = Param(
-            self.regional_global_indices, self.tech_or_group, self.operator
+            self.regional_global_indices, self.tech_or_group, self.operator, within=Any
         )
         self.limit_growth_new_capacity = Param(
-            self.regional_global_indices, self.tech_or_group, self.operator
+            self.regional_global_indices, self.tech_or_group, self.operator, within=Any
         )
         self.limit_degrowth_new_capacity = Param(
-            self.regional_global_indices, self.tech_or_group, self.operator
+            self.regional_global_indices, self.tech_or_group, self.operator, within=Any
         )
         self.limit_growth_new_capacity_delta = Param(
-            self.regional_global_indices, self.tech_or_group, self.operator
+            self.regional_global_indices, self.tech_or_group, self.operator, within=Any
         )
         self.limit_degrowth_new_capacity_delta = Param(
-            self.regional_global_indices, self.tech_or_group, self.operator
+            self.regional_global_indices, self.tech_or_group, self.operator, within=Any
         )
 
         self.limit_emission_constraint_rpe = Set(

temoa/data_io/component_manifest.py

@@ -288,6 +288,14 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]:
             is_period_filtered=False,  # Custom loader handles all logic
             is_table_required=False,
         ),
+        LoadItem(
+            component=model.retired_existing_capacity,
+            table='output_retired_capacity',
+            columns=['region', 'period', 'tech', 'vintage', 'cap_early'],
+            custom_loader_name='_load_retired_existing_capacity',
+            is_period_filtered=False,  # Custom loader handles all logic
+            is_table_required=False,
+        ),
         LoadItem(
             component=model.cost_invest,
             table='cost_invest',
@@ -419,26 +427,23 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]:
             component=model.lifetime_tech,
             table='lifetime_tech',
             columns=['region', 'tech', 'lifetime'],
-            validator_name='viable_rt',
-            validation_map=(0, 1),
+            custom_loader_name='_load_lifetime_tech',
             is_period_filtered=False,
             is_table_required=False,
         ),
         LoadItem(
             component=model.lifetime_process,
             table='lifetime_process',
             columns=['region', 'tech', 'vintage', 'lifetime'],
-            validator_name='viable_rtv',
-            validation_map=(0, 1, 2),
+            custom_loader_name='_load_lifetime_process',
             is_period_filtered=False,
             is_table_required=False,
         ),
         LoadItem(
             component=model.lifetime_survival_curve,
             table='lifetime_survival_curve',
             columns=['region', 'period', 'tech', 'vintage', 'fraction'],
-            validator_name='viable_rtv',
-            validation_map=(0, 2, 3),
+            custom_loader_name='_load_lifetime_survival_curve',
             is_period_filtered=False,
             is_table_required=False,
         ),
@@ -649,6 +654,66 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]:
             validation_map=(0, 1, 2),
             is_table_required=False,
         ),
+        LoadItem(
+            component=model.limit_growth_capacity,
+            table='limit_growth_capacity',
+            columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'],
+            index_length=3,
+            validator_name='viable_rt',
+            validation_map=(0, 1),
+            is_period_filtered=False,
+            is_table_required=False,
+        ),
+        LoadItem(
+            component=model.limit_growth_new_capacity,
+            table='limit_growth_new_capacity',
+            columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'],
+            index_length=3,
+            validator_name='viable_rt',
+            validation_map=(0, 1),
+            is_period_filtered=False,
+            is_table_required=False,
+        ),
+        LoadItem(
+            component=model.limit_growth_new_capacity_delta,
+            table='limit_growth_new_capacity_delta',
+            columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'],
+            index_length=3,
+            validator_name='viable_rt',
+            validation_map=(0, 1),
+            is_period_filtered=False,
+            is_table_required=False,
+        ),
+        LoadItem(
+            component=model.limit_degrowth_capacity,
+            table='limit_degrowth_capacity',
+            columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'],
+            index_length=3,
+            validator_name='viable_rt',
+            validation_map=(0, 1),
+            is_period_filtered=False,
+            is_table_required=False,
+        ),
+        LoadItem(
+            component=model.limit_degrowth_new_capacity,
+            table='limit_degrowth_new_capacity',
+            columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'],
+            index_length=3,
+            validator_name='viable_rt',
+            validation_map=(0, 1),
+            is_period_filtered=False,
+            is_table_required=False,
+        ),
+        LoadItem(
+            component=model.limit_degrowth_new_capacity_delta,
+            table='limit_degrowth_new_capacity_delta',
+            columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'],
+            index_length=3,
+            validator_name='viable_rt',
+            validation_map=(0, 1),
+            is_period_filtered=False,
+            is_table_required=False,
+        ),
         LoadItem(
             component=model.limit_resource,
             table='limit_resource',