merge

Author: SimonBlanke

.github/workflows/test.yml

@@ -42,6 +42,9 @@ jobs:
 
           make install-no-extras-for-test
 
+      - name: Show dependencies
+        run: python -m pip list
+
       - name: Test with pytest
         run: |
           python -m pytest tests -p no:warnings
@@ -77,6 +80,9 @@ jobs:
 
           make install-all-extras-for-test
 
+      - name: Show dependencies
+        run: python -m pip list
+
       - name: Test with pytest
         run: |
           python -m pytest tests --cov=hyperactive --cov-report=term-missing --cov-report=xml -p no:warnings
@@ -86,7 +92,7 @@ jobs:
           python -m pytest src/hyperactive -p no:warnings
 
   test-sklearn-versions:
-    name: test-sklearn-${{ matrix.sklearn-version }} python-${{ matrix.python-version }}
+    name: test-sklearn-${{ matrix.sklearn-version }}-python-${{ matrix.python-version }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -98,18 +104,19 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python 3.12
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies for scikit-learn ${{ matrix.sklearn-version }}
         run: |
           python -m pip install --upgrade pip
-          python -m pip install build pytest
-          make install
-          python -m pip install scikit-learn==${{ matrix.sklearn-version }}
+          python -m pip install .[all_extras,test] scikit-learn==${{ matrix.sklearn-version }}
+
+      - name: Show dependencies
+        run: python -m pip list
 
       - name: Run sklearn integration tests for ${{ matrix.sklearn-version }}
         run: |
-          python -m pytest -x -p no:warnings tests/integrations/sklearn/
+          python -m pytest -x -p no:warnings tests/integrations/sklearn/

extension_templates/experiments.py

@@ -75,7 +75,11 @@ class MyExperiment(BaseExperiment):
         #
         "property:randomness": "random",
         # valid values: "random", "deterministic"
-        # if "deterministic", two calls of score must result in the same value
+        # if "deterministic", two calls of "evaluate" must result in the same value
+        #
+        "property:higher_or_lower_is_better": "lower",
+        # valid values: "higher", "lower", "mixed"
+        # whether higher or lower returns of "evaluate" are better
         #
         # --------------
         # packaging info
@@ -147,25 +151,25 @@ def _paramnames(self):
         return ["score_param1", "score_param2"]
 
     # todo: implement this, mandatory
-    def _score(self, params):
-        """Score the parameters.
+    def _evaluate(self, params):
+        """Evaluate the parameters.
 
         Parameters
         ----------
         params : dict with string keys
-            Parameters to score.
+            Parameters to evaluate.
 
         Returns
         -------
         float
-            The score of the parameters.
+            The value of the parameters as per evaluation.
         dict
             Additional metadata about the search.
         """
         # params is a dictionary with keys being paramnames or subset thereof
         # IMPORTANT: avoid side effects to params!
         #
-        # the method may work if only a subste of the parameters in paramnames is passed
+        # the method may work if only a subset of the parameters in paramnames is passed
         # but this is not necessary
         value = 42  # must be numpy.float64
         metadata = {"some": "metadata"}  # can be any dict
@@ -230,18 +234,19 @@ def get_test_params(cls, parameter_set="default"):
 
     @classmethod
     def _get_score_params(self):
-        """Return settings for testing the score function. Used in tests only.
+        """Return settings for testing score/evaluate functions. Used in tests only.
 
-        Returns a list, the i-th element corresponds to self.get_test_params()[i].
-        It should be a valid call for self.score.
+        Returns a list, the i-th element should be valid arguments for
+        self.evaluate and self.score, of an instance constructed with
+        self.get_test_params()[i].
 
         Returns
         -------
         list of dict
             The parameters to be used for scoring.
         """
         # dict keys should be same as paramnames return
-        # or subset, only if _score allows for subsets of parameters
+        # or subset, only if _evaluate allows for subsets of parameters
         score_params1 = {"score_param1": "foo", "score_param2": "bar"}
         score_params2 = {"score_param1": "baz", "score_param2": "qux"}
         return [score_params1, score_params2]

extension_templates/optimizers.py

@@ -142,7 +142,8 @@ def _paramnames(self):
         return ["score_param1", "score_param2"]
 
     # optional: implement this to prepare arguments for _run
-    # the default is all parameters passed to __init__, except ex
+    # the default is all parameters passed to __init__, minus the experiment
+    # the result of this is passed to _run as search_config
     def get_search_config(self):
         """Get the search configuration.
 
@@ -153,12 +154,15 @@ def get_search_config(self):
         """
         # the default
         search_config = super().get_search_config()
+        # example of adding a new parameter to the search config
+        # this is optional, but can be useful for clean separation or API interfacing
         search_config["one_more_param"] = 42
+        # this return is available in _run as search_config
         return search_config
 
     # todo: implement this, mandatory
     def _run(self, experiment, **search_config):
-        """Run the optimization search process.
+        """Run the optimization search process to maximize the experiment's score.
 
         Parameters
         ----------
@@ -173,6 +177,8 @@ def _run(self, experiment, **search_config):
             The best parameters found during the search.
             Must have keys a subset or identical to experiment.paramnames().
         """
+        # important: the search logic should *maximize* the experiment's score
+        # this is the main method to implement, it should return the best parameters
         best_params = {"write_some_logic_to_get": "best_params"}
         return best_params
 

pyproject.toml

@@ -42,20 +42,20 @@ dependencies = [
   "pandas <3.0.0",
   "gradient-free-optimizers >=1.2.4, <2.0.0",
   "scikit-base <1.0.0",
-  "scikit-learn <1.7.0",
+  "scikit-learn <1.8.0",
 ]
 
 [project.optional-dependencies]
 sklearn-integration = [
-  "scikit-learn <1.7.0",
+  "scikit-learn <1.8.0",
 ]
 build = [
   "setuptools",
   "build",
   "wheel",
 ]
 test = [
-  "pytest == 8.4.0",
+  "pytest == 8.4.1",
   "flake8",
   "pytest-cov",
   "pathos",

requirements/requirements-test.in

@@ -1,4 +1,4 @@
-pytest == 8.4.0
+pytest == 8.4.1
 flake8
 pytest-cov
 pathos

src/hyperactive/base/_experiment.py

@@ -14,13 +14,15 @@ class BaseExperiment(BaseObject):
         "property:randomness": "random",  # random or deterministic
         # if deterministic, two calls of score will result in the same value
         # random = two calls may result in different values; same as "stochastic"
+        "property:higher_or_lower_is_better": "higher",  # "higher", "lower", "mixed"
+        # whether higher or lower scores are better
     }
 
     def __init__(self):
         super().__init__()
 
     def __call__(self, **kwargs):
-        """Score parameters, with kwargs call."""
+        """Score parameters, with kwargs call. Same as score call."""
         score, _ = self.score(kwargs)
         return score
 
@@ -48,30 +50,55 @@ def _paramnames(self):
         """
         raise NotImplementedError
 
-    def score(self, params):
-        """Score the parameters.
+    def evaluate(self, params):
+        """Evaluate the parameters.
 
         Parameters
         ----------
         params : dict with string keys
-            Parameters to score.
+            Parameters to evaluate.
 
         Returns
         -------
         float
-            The score of the parameters.
+            The value of the parameters as per evaluation.
         dict
             Additional metadata about the search.
         """
         paramnames = self.paramnames()
         if not set(params.keys()) <= set(paramnames):
             raise ValueError("Parameters do not match.")
-        res, metadata = self._score(params)
+        res, metadata = self._evaluate(params)
         res = np.float64(res)
         return res, metadata
 
-    def _score(self, params):
-        """Score the parameters.
+    def _evaluate(self, params):
+        """Evaluate the parameters.
+
+        Parameters
+        ----------
+        params : dict with string keys
+            Parameters to evaluate.
+
+        Returns
+        -------
+        float
+            The value of the parameters as per evaluation.
+        dict
+            Additional metadata about the search.
+        """
+        raise NotImplementedError
+
+    def score(self, params):
+        """Score the parameters - with sign such that higher is always better.
+
+        Same as ``evaluate`` call except for the sign chosen so that higher is better.
+
+        If the tag ``property:higher_or_lower_is_better`` is set to
+        ``"lower"``, the result is ``-self.evaluate(params)``.
+
+        If the tag is set to ``"higher"``, the result is
+        identical to ``self.evaluate(params)``.
 
         Parameters
         ----------
@@ -85,4 +112,14 @@ def _score(self, params):
         dict
             Additional metadata about the search.
         """
-        raise NotImplementedError
+        hib = self.get_tag("property:higher_or_lower_is_better", "lower")
+        if hib == "higher":
+            sign = 1
+        elif hib == "lower":
+            sign = -1
+
+        eval_res = self.evaluate(params)
+        value = eval_res[0]
+        metadata = eval_res[1]
+
+        return sign * value, metadata

src/hyperactive/base/_optimizer.py

@@ -52,12 +52,23 @@ def get_experiment(self):
         return self._experiment
 
     def run(self):
-        """Run the optimization search process.
+        """Run the optimization search process to maximize the experiment's score.
+
+        The optimization searches for a maximizer of the experiment's
+        ``score`` method.
+
+        Depending on the tag ``property:higher_or_lower_is_better`` being
+        set to ``higher`` or ``lower``, the ``run`` method will search for:
+
+        * the minimizer of the ``evaluate`` method if the tag is ``lower``
+        * the maximizer of the ``evaluate`` method if the tag is ``higher``
 
         Returns
         -------
         best_params : dict
             The best parameters found during the optimization process.
+            The dict ``best_params`` can be used in ``experiment.score`` or
+            ``experiment.evaluate`` directly.
         """
         experiment = self.get_experiment()
         search_config = self.get_search_config()