feat: Extend evaluation_time to problems

Author: olafmersmann

SCHEMA.md

@@ -91,7 +91,7 @@ There are separate files which contain curated usage examples for problems or su
 | `description`     | str                               | required                                     |
 | `language`        | str? (e.g. `python`, `c`)         |                                              |
 | `links`           | list of [Link](#link)?            | repo, release, docs…                         |
-| `evaluation_time` | str?                              | free-form ("8 minutes", "fast")              |
+| `evaluation_time` | set of str?                       | free-form list ("8 minutes", "fast")         |
 | `requirements`    | str or list of str?               | URL to requirements file or list of packages |
 
 ```yaml
@@ -135,11 +135,15 @@ The schema deliberately puts most descriptive fields here so suites can be chara
 | `modality`                               | set of str?                                    | `{"unimodal"}`, `{"multimodal"}`                   |
 | `fidelity_levels`                        | set of int?                                    | `{1}` = single-fidelity, `{1,2}` = multi-fidelity  |
 | `code_examples`                          | set of str?                                    | paths to example scripts                           |
+| `evaluation_time`                        | set of str?                                    | free-form list ("8 minutes", "fast")         |
 | `source`                                 | set of str?                                    | `{"artificial"}`, `{"real-world"}`                 |
 
 > `objectives` is a set of integers because we don't assume extreme scalability in this property so explicit enumeration is fine.
 > Dimensions of variables on the other hand are ranges because here problems often are scalable over wide ranges.
 
+When no `evaluation_time` is set, it percolates up from any referenced implementations.
+The same is true for the `variables` and `constraints` properties of a suite that has references to problems.
+
 ### Problem
 
 One optimization problem (possibly parameterised by instances).

src/opltools/schema.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import Any
 from typing_extensions import Self
 from pydantic import BaseModel, RootModel, ConfigDict, model_validator
 
@@ -98,7 +99,7 @@ class Implementation(Thing):
     description: str
     links: list[Link] | None = None
     language: str | None = None
-    evaluation_time: str | None = None
+    evaluation_time: set[str] | None = None
     requirements: str | list[str] | None = None
 
 
@@ -118,6 +119,7 @@ class ProblemLike(Thing):
     can_evaluate_objectives_independently: YesNoSome | None = None
     modality: set[str] | None = None
     fidelity_levels: set[int] | None = None
+    evaluation_time: set[str] | None = None
     code_examples: set[str] | None = None
     source: set[str] | None = None
 
@@ -152,35 +154,32 @@ def _check_id_references(self, ids, type: OPLType) -> None:
             else:
                 raise ValueError(f"Missing {type.name} with id '{id}'")
 
-    # For a given suite, make sure the fidelty_levels property contains
-    # the fidelity_levels of all problems in the suite.
-    def _fixup_suite_fidelity(self, suite: Suite):
-        if suite.problems:
-            if not suite.fidelity_levels:
-                suite.fidelity_levels = set()
-            for pid in suite.problems:
-                problem = self.root[pid]
-                assert isinstance(problem, Problem)
-                if problem.fidelity_levels:
-                    suite.fidelity_levels.update(problem.fidelity_levels)
-
-        return suite
-
-    def _fixup_suite_variables(self, suite: Suite):
-        if not suite.problems:
+    def _percolate_set(self, thing: Any, children: set | None, property: str):
+        """Propagate some `property` from child objects to the parent by calculating the union of all the child property sets.
+
+        This is useful to propagate properties like `variables`, `constraints`, etc. from problems up to the suite.
+        """
+        if children is None:
             return
 
-        if suite.variables is None:
-            suite.variables = set()
-        for pid in suite.problems:
-            problem = self.root[pid]
-            assert isinstance(problem, Problem)
-            if problem.variables is not None:
-                suite.variables.update(problem.variables)
+        if getattr(thing, property, None) is None:
+            setattr(thing, property, set())
+        thing_set = getattr(thing, property)
+
+        for child_id in children:
+            child = self.root[child_id]
+            child_set = getattr(child, property, None)
+            if child_set is not None:
+                thing_set.update(child_set)
 
     @model_validator(mode="after")
     def _validate(self) -> Self:
-        # Make sure all problems referenced in suites exists
+        # First check and fixup all problems
+        for id, thing in self.root.items():
+            if isinstance(thing, Problem) and thing.implementations:
+                self._percolate_set(thing, thing.implementations, "evaluation_time")
+
+        # Then check and fixup all suites because changes from the problems need to propagate to the suites
         for id, thing in self.root.items():
             if isinstance(thing, Suite) and thing.problems:
                 for problem_id in thing.problems:
@@ -192,7 +191,12 @@ def _validate(self) -> Self:
                         raise ValueError(
                             f"Suite {id} references problem with id '{problem_id}' but id is a {self.root[problem_id].type.name}."
                         )
-                self._fixup_suite_fidelity(thing)
+
+                self._percolate_set(thing, thing.problems, "fidelity_levels")
+                self._percolate_set(thing, thing.problems, "variables")
+                self._percolate_set(thing, thing.problems, "constraints")
+                self._percolate_set(thing, thing.problems, "evaluation_time")
+
         return self