sign

Author: fkiraly

extension_templates/experiments.py

@@ -77,6 +77,10 @@ class MyExperiment(BaseExperiment):
         # valid values: "random", "deterministic"
         # if "deterministic", two calls of score must result in the same value
         #
+        "property:higher_or_lower_is_better": "lower",
+        # valid values: "higher", "lower", "mixed"
+        # whether higher or lower scores are better
+        #
         # --------------
         # packaging info
         # --------------

src/hyperactive/base/_experiment.py

@@ -14,14 +14,16 @@ class BaseExperiment(BaseObject):
         "property:randomness": "random",  # random or deterministic
         # if deterministic, two calls of score will result in the same value
         # random = two calls may result in different values; same as "stochastic"
+        "property:higher_or_lower_is_better": "lower",  # "higher", "lower", "mixed"
+        # whether higher or lower scores are better
     }
 
     def __init__(self):
         super().__init__()
 
     def __call__(self, **kwargs):
-        """Score parameters, with kwargs call."""
-        score, _ = self.score(kwargs)
+        """Score parameters, with kwargs call. Same as cost call."""
+        score, _ = self.cost(kwargs)
         return score
 
     @property
@@ -86,3 +88,36 @@ def _score(self, params):
             Additional metadata about the search.
         """
         raise NotImplementedError
+
+    def cost(self, params):
+        """Score the parameters - with sign such that lower is better.
+
+        Same as ``score`` call except for the sign.
+
+        If the tag ``property:higher_or_lower_is_better`` is set to
+        ``"higher"``, the result is ``-self.score(params)``.
+
+        If the tag is set to ``"lower"``, the result is
+        identical to ``self.score(params)``.
+
+        Parameters
+        ----------
+        params : dict with string keys
+            Parameters to score.
+
+        Returns
+        -------
+        float
+            The score of the parameters.
+        dict
+            Additional metadata about the search.
+        """
+        hib = self.get_tag("property:higher_or_lower_is_better", "lower")
+        if hib == "higher":
+            sign = -1
+        elif hib == "lower":
+            sign = 1
+
+        score_res = self.score(params)
+
+        return sign * score_res[0], score_res[1]

src/hyperactive/experiment/integrations/sklearn_cv.py

@@ -110,6 +110,13 @@ def __init__(self, estimator, X, y, scoring=None, cv=None):
                 self._scoring = make_scorer(scoring)
         self.scorer_ = self._scoring
 
+        # Set the sign of the scoring function
+        if hasattr(self._scoring, "_score"):
+            score_func = self._scoring._score_func
+            _sign = _guess_sign_of_sklmetric(score_func)
+            _sign_str = "higher" if _sign == 1 else "lower"
+            self.set_tags(**{"property:higher_or_lower_is_better": _sign_str})
+
     def _paramnames(self):
         """Return the parameter names of the search.
 
@@ -235,3 +242,80 @@ def _get_score_params(self):
         score_params_regress = {"C": 1.0, "kernel": "linear"}
         score_params_defaults = {"C": 1.0, "kernel": "linear"}
         return [score_params_classif, score_params_regress, score_params_defaults]
+
+
+def _guess_sign_of_sklmetric(scorer):
+    """Guess the sign of a sklearn metric scorer.
+
+    Parameters
+    ----------
+    scorer : callable
+        The sklearn metric scorer to guess the sign for.
+
+    Returns
+    -------
+    int
+        1 if higher scores are better, -1 if lower scores are better.
+    """
+    HIGHER_IS_BETTER = {
+        # Classification
+        "accuracy_score": True,
+        "auc": True,
+        "average_precision_score": True,
+        "balanced_accuracy_score": True,
+        "brier_score_loss": False,
+        "class_likelihood_ratios": False,
+        "cohen_kappa_score": True,
+        "d2_log_loss_score": True,
+        "dcg_score": True,
+        "f1_score": True,
+        "fbeta_score": True,
+        "hamming_loss": False,
+        "hinge_loss": False,
+        "jaccard_score": True,
+        "log_loss": False,
+        "matthews_corrcoef": True,
+        "ndcg_score": True,
+        "precision_score": True,
+        "recall_score": True,
+        "roc_auc_score": True,
+        "top_k_accuracy_score": True,
+        "zero_one_loss": False,
+
+        # Regression
+        "d2_absolute_error_score": True,
+        "d2_pinball_score": True,
+        "d2_tweedie_score": True,
+        "explained_variance_score": True,
+        "max_error": False,
+        "mean_absolute_error": False,
+        "mean_absolute_percentage_error": False,
+        "mean_gamma_deviance": False,
+        "mean_pinball_loss": False,
+        "mean_poisson_deviance": False,
+        "mean_squared_error": False,
+        "mean_squared_log_error": False,
+        "mean_tweedie_deviance": False,
+        "median_absolute_error": False,
+        "r2_score": True,
+        "root_mean_squared_error": False,
+        "root_mean_squared_log_error": False,
+    }
+
+    scorer_name = getattr(scorer, "__name__", None)
+
+    if hasattr(scorer, "greater_is_better"):
+        return 1 if scorer.greater_is_better else -1
+    elif scorer_name in HIGHER_IS_BETTER:
+        return 1 if HIGHER_IS_BETTER[scorer_name] else -1
+    elif scorer_name.endswith("_score"):
+        # If the scorer name ends with "_score", we assume higher is better
+        return 1
+    elif scorer_name.endswith("_loss") or scorer_name.endswith("_deviance"):
+        # If the scorer name ends with "_loss", we assume lower is better
+        return -1
+    elif scorer_name.endswith("_error"):
+        return -1
+    else:
+        # If we cannot determine the sign, we assume lower is better
+        return -1

src/hyperactive/opt/_adapters/_gfo.py

@@ -133,7 +133,7 @@ def _run(self, experiment, **search_config):
 
         with StdoutMute(active=not self.verbose):
             gfopt.search(
-                objective_function=experiment.score,
+                objective_function=experiment.cost,
                 n_iter=n_iter,
                 max_time=max_time,
             )

src/hyperactive/tests/test_all_objects.py

@@ -45,6 +45,7 @@ class PackageConfig:
         "maintainers",
         # experiments
         "property:randomness",
+        "property:higher_or_lower_is_better",
         # optimizers
         "info:name",  # str
         "info:local_vs_global",  # "local", "mixed", "global"
@@ -184,10 +185,19 @@ def test_score_function(self, object_class):
             assert isinstance(score, float), f"Score is not a float: {score}"
             assert isinstance(metadata, dict), f"Metadata is not a dict: {metadata}"
 
+            cost_res = inst.cost(obj)
+            msg = f"Cost function did not return a length two tuple: {res}"
+            assert isinstance(cost_res, tuple) and len(cost_res) == 2, msg
+            c_score, c_metadata = cost_res
+            assert isinstance(c_score, float), f"Score is not a float: {c_score}"
+            assert isinstance(c_metadata, dict), f"Metadata is not a dict: {c_metadata}"
+
+            assert abs(c_score) == score
+
             call_sc = inst(**obj)
             assert isinstance(call_sc, float), f"Score is not a float: {call_sc}"
             if inst.get_tag("property:randomness") == "deterministic":
-                assert score == call_sc, f"Score does not match: {score} != {call_sc}"
+                assert c_score == call_sc, f"Score does not match: {score} != {call_sc}"
 
 
 class OptimizerFixtureGenerator(BaseFixtureGenerator):