From 1cf9bd25d98b3387128fc4d8dec054c25186c831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Tue, 22 Oct 2024 16:33:40 +0200 Subject: [PATCH 01/62] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 155182c5..59c26f5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "tqdm >=4.48.0, <5.0.0", "pandas <3.0.0", "gradient-free-optimizers >=1.2.4, <2.0.0", + "scikit-base <1.0.0", ] [project.optional-dependencies] From 40fe1926162bda553b8311ef8f70915bf2a93354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Wed, 30 Oct 2024 14:35:49 +0100 Subject: [PATCH 02/62] base classes and sklearn --- src/hyperactive/base/__init__.py | 6 ++ src/hyperactive/base/_experiment.py | 71 +++++++++++++++++++ src/hyperactive/base/_optimizer.py | 33 +++++++++ .../sklearn/sklearn_cv_experiment.py | 59 +++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 src/hyperactive/base/__init__.py create mode 100644 src/hyperactive/base/_experiment.py create mode 100644 src/hyperactive/base/_optimizer.py create mode 100644 src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py diff --git a/src/hyperactive/base/__init__.py b/src/hyperactive/base/__init__.py new file mode 100644 index 00000000..bf0307d4 --- /dev/null +++ b/src/hyperactive/base/__init__.py @@ -0,0 +1,6 @@ +"""Base classes for optimizers and experiments.""" + +from hyperactive.base._experiment import BaseExperiment +from hyperactive.base._optimizer import BaseOptimizer + +__all__ = ["BaseExperiment", "BaseOptimizer"] diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py new file mode 100644 index 00000000..ad3a5999 --- /dev/null +++ b/src/hyperactive/base/_experiment.py @@ -0,0 +1,71 @@ +"""Base class for experiment.""" + +from skbase.base import BaseObject + + +class BaseExperiment(BaseObject): + """Base class for experiment.""" + + def __init__(self): + super().__init__() + + def __call__(self, **kwargs): + """Score parameters, with kwargs call.""" + return self.score(kwargs) + + def paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return self._paramnames() + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + raise NotImplementedError + + def score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + paramnames = self.paramnames() + if not set(paramnames) == set(params.keys()): + raise ValueError("Parameters do not match.") + return self._score(**params) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + raise NotImplementedError diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py new file mode 100644 index 00000000..55bf7d1a --- /dev/null +++ b/src/hyperactive/base/_optimizer.py @@ -0,0 +1,33 @@ +"""Base class for optimizer.""" + +from skbase.base import BaseObject + + +class BaseOptimizer(BaseObject): + """Base class for optimizer.""" + + def __init__(self): + super().__init__() + + def add_search(self, experiment, search_config: dict): + """Add a new optimization search process with specified parameters. + + Parameters + ---------- + experiment : BaseExperiment + The experiment to optimize parameters for. + search_config : dict with str keys + The search configuration dictionary. + """ + self.experiment = experiment + self.search_config = search_config + + def run(self, max_time=None): + """Run the optimization search process. + + Parameters + ---------- + max_time : float + The maximum time used for the optimization process. + """ + raise NotImplementedError diff --git a/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py b/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py new file mode 100644 index 00000000..c3853ea6 --- /dev/null +++ b/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py @@ -0,0 +1,59 @@ +"""Experiment adapter for sklearn cross-validation experiments.""" + +from sklearn import clone +from sklearn.model_selection import cross_validate +from sklearn.utils.validation import _num_samples + +from hyperactive.base import BaseExperiment + +class BaseExperiment(BaseExperiment): + + def __init__(self, estimator, scoring, cv, X, y): + self.estimator = estimator + self.X = X + self.y = y + self.scoring = scoring + self.cv = cv + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return list(self.estimator.get_params().keys()) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + estimator = clone(self.estimator) + estimator.set_params(**params) + + cv_results = cross_validate( + self.estimator, + self.X, + self.y, + cv=self.cv, + ) + + add_info_d = { + "score_time": cv_results["score_time"], + "fit_time": cv_results["fit_time"], + "n_test_samples": _num_samples(self.X), + } + + return cv_results["test_score"].mean(), add_info_d From 606ba6290b4bba3d0ba139a883eeedbd141528bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Wed, 30 Oct 2024 14:40:09 +0100 Subject: [PATCH 03/62] Update sklearn_cv_experiment.py --- src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py b/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py index c3853ea6..b77aaf7d 100644 --- a/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py +++ b/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py @@ -6,7 +6,7 @@ from hyperactive.base import BaseExperiment -class BaseExperiment(BaseExperiment): +class SklearnCvExperiment(BaseExperiment): def __init__(self, estimator, scoring, cv, X, y): self.estimator = estimator From 34581ce18f25bbf40dae79e50419a9d600510a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Wed, 30 Oct 2024 14:58:37 +0100 Subject: [PATCH 04/62] refactor integration --- .../integrations/sklearn/hyperactive_search_cv.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py b/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py index ad4e5d5e..a7db6550 100644 --- a/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py +++ b/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py @@ -18,6 +18,7 @@ from .best_estimator import BestEstimator as _BestEstimator_ from .checks import Checks from ...optimizers import RandomSearchOptimizer +from hyperactive.integrations.sklearn.sklearn_cv_experiment import SklearnCvExperiment class HyperactiveSearchCV(BaseEstimator, _BestEstimator_, Checks): @@ -118,12 +119,14 @@ def fit(self, X, y, **fit_params): fit_params = _check_method_params(X, params=fit_params) self.scorer_ = check_scoring(self.estimator, scoring=self.scoring) - objective_function_adapter = ObjectiveFunctionAdapter( - self.estimator, + experiment = SklearnCvExperiment( + estimator=self.estimator, + scoring=self.scorer_, + cv=self.cv, + X=X, + y=y, ) - objective_function_adapter.add_dataset(X, y) - objective_function_adapter.add_validation(self.scorer_, self.cv) - objective_function = objective_function_adapter.objective_function + objective_function = experiment.score hyper = Hyperactive(verbosity=False) hyper.add_search( From 3bb00444430d7f4326c5d10a9dea3e07a2111619 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 12 Jan 2025 19:15:54 +0100 Subject: [PATCH 05/62] add API-design draft for optimizer, experiment and search-space --- examples/v5_API_example/experiments/keras.py | 47 +++++++++++++++++++ .../v5_API_example/experiments/sklearn.py | 35 ++++++++++++++ .../experiments/test_function.py | 31 ++++++++++++ examples/v5_API_example/optimizer.py | 40 ++++++++++++++++ .../v5_API_example/search_space_optional.py | 16 +++++++ 5 files changed, 169 insertions(+) create mode 100644 examples/v5_API_example/experiments/keras.py create mode 100644 examples/v5_API_example/experiments/sklearn.py create mode 100644 examples/v5_API_example/experiments/test_function.py create mode 100644 examples/v5_API_example/optimizer.py create mode 100644 examples/v5_API_example/search_space_optional.py diff --git a/examples/v5_API_example/experiments/keras.py b/examples/v5_API_example/experiments/keras.py new file mode 100644 index 00000000..511976bb --- /dev/null +++ b/examples/v5_API_example/experiments/keras.py @@ -0,0 +1,47 @@ +from tensorflow import keras +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split + +from hyperactive import BaseExperiment + + +X, y = make_classification(n_samples=1000, n_features=20, random_state=42) +X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2) + + +class KerasMultiLayerPerceptron(BaseExperiment): + def __init__(self, X_train, X_val, y_train, y_val): + super().__init__() + + self.X_train = X_train + self.X_val = X_val + self.y_train = y_train + self.y_val = y_val + + def _score(self, **params): + dense_layer_0 = params["dense_layer_0"] + activation_layer_0 = params["activation_layer_0"] + + model = keras.Sequential( + [ + keras.layers.Dense( + dense_layer_0, + activation=activation_layer_0, + input_shape=(20,), + ), + keras.layers.Dense(1, activation="sigmoid"), + ] + ) + model.compile( + optimizer=keras.optimizers.Adam(learning_rate=0.01), + loss="binary_crossentropy", + metrics=["accuracy"], + ) + model.fit( + self.X_train, + self.y_train, + batch_size=32, + epochs=10, + validation_data=(self.X_val, self.y_val), + ) + return model.evaluate(X_val, y_val)[1] diff --git a/examples/v5_API_example/experiments/sklearn.py b/examples/v5_API_example/experiments/sklearn.py new file mode 100644 index 00000000..d925167f --- /dev/null +++ b/examples/v5_API_example/experiments/sklearn.py @@ -0,0 +1,35 @@ +from sklearn.model_selection import cross_val_score +from sklearn.ensemble import GradientBoostingRegressor + + +from hyperactive import BaseExperiment + + +class SklearnExperiment(BaseExperiment): + def __init__(self, estimator, X, y, cv=4): + super().__init__() + + self.estimator = estimator + self.X = X + self.y = y + self.cv = cv + + def _score(self, **params): + model = self.estimator(**params) + scores = cross_val_score(model, self.X, self.y, cv=self.cv) + return scores.mean() + + +class GradientBoostingExperiment(BaseExperiment): + def __init__(self, X, y, cv=4): + super().__init__() + + self.estimator = GradientBoostingRegressor # The user could also predefine the estimator + self.X = X + self.y = y + self.cv = cv + + def _score(self, **params): + model = self.estimator(**params) + scores = cross_val_score(model, self.X, self.y, cv=self.cv) + return scores.mean() diff --git a/examples/v5_API_example/experiments/test_function.py b/examples/v5_API_example/experiments/test_function.py new file mode 100644 index 00000000..9260bfc4 --- /dev/null +++ b/examples/v5_API_example/experiments/test_function.py @@ -0,0 +1,31 @@ +import numpy as np + +from hyperactive import BaseExperiment + + +class SphereFunction(BaseExperiment): + def __init__(self, const, n_dim=2): + super().__init__() + + self.const = const + self.n_dim = n_dim + + def _score(self, **params): + return np.sum(np.array(params) ** 2) + self.const + + +class AckleyFunction(BaseExperiment): + def __init__(self, A): + super().__init__() + self.A = A + + def _score(self, **params): + x = params["x0"] + y = params["x1"] + + loss1 = -self.A * np.exp(-0.2 * np.sqrt(0.5 * (x * x + y * y))) + loss2 = -np.exp(0.5 * (np.cos(2 * np.pi * x) + np.cos(2 * np.pi * y))) + loss3 = np.exp(1) + loss4 = self.A + + return -(loss1 + loss2 + loss3 + loss4) diff --git a/examples/v5_API_example/optimizer.py b/examples/v5_API_example/optimizer.py new file mode 100644 index 00000000..8660599f --- /dev/null +++ b/examples/v5_API_example/optimizer.py @@ -0,0 +1,40 @@ +import numpy as np +from sklearn.datasets import load_diabetes +from sklearn.tree import DecisionTreeRegressor + +from hyperactive.optimizers import ( + HillClimbingOptimizer, + RandomRestartHillClimbingOptimizer, +) + +from .experiments.test_function import SklearnExperiment +from .search_space_optional import SearchSpace + + +data = load_diabetes() +X, y = data.data, data.target + + +search_space = { + "max_depth": list(np.arange(2, 15, 1)), + "min_samples_split": list(np.arange(2, 25, 2)), +} + +""" optional way of defining search-space +search_space = SearchSpace( + max_depth=list(np.arange(2, 15, 1)), + min_samples_split=list(np.arange(2, 25, 2)), +) +""" + +experiment = SklearnExperiment(DecisionTreeRegressor, X, y, cv=4) + +optimizer1 = HillClimbingOptimizer(n_iter=50) +optimizer2 = RandomRestartHillClimbingOptimizer(n_iter=50, n_jobs=2) + +optimizer1.add_search(experiment, search_space) +optimizer2.add_search(experiment, search_space) + +hyper = optimizer1 + optimizer2 + +hyper.run(max_time=5) diff --git a/examples/v5_API_example/search_space_optional.py b/examples/v5_API_example/search_space_optional.py new file mode 100644 index 00000000..ef7271a9 --- /dev/null +++ b/examples/v5_API_example/search_space_optional.py @@ -0,0 +1,16 @@ +import numpy as np +from hyperactive import BaseSearchSpace + + +class SearchSpace: + search_space: dict = None + + def __init__(self, **params): + super().__init__() + + for key, value in params.items(): + setattr(self, key, value) + self.search_space[key] = value + + def __call__(self): + return self.search_space From 9dcc02537b0879fe5ec53497d2470e66d5af4457 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Wed, 15 Jan 2025 20:29:34 +0100 Subject: [PATCH 06/62] add docstring and explanations --- examples/v5_API_example/experiments/keras.py | 21 ++++++++++++ .../v5_API_example/experiments/sklearn.py | 32 +++++++++++++++++++ .../experiments/test_function.py | 29 +++++++++++++++++ examples/v5_API_example/optimizer.py | 1 + .../v5_API_example/search_space_optional.py | 9 ++++++ 5 files changed, 92 insertions(+) diff --git a/examples/v5_API_example/experiments/keras.py b/examples/v5_API_example/experiments/keras.py index 511976bb..dce2441b 100644 --- a/examples/v5_API_example/experiments/keras.py +++ b/examples/v5_API_example/experiments/keras.py @@ -10,6 +10,27 @@ class KerasMultiLayerPerceptron(BaseExperiment): + """ + A class for creating and evaluating a Keras-based Multi-Layer Perceptron (MLP) model. + + This class inherits from BaseExperiment and is designed to build a simple MLP + using Keras, compile it with the Adam optimizer, and train it on the provided + training data. The model consists of one hidden dense layer with configurable + size and activation function, followed by an output layer with a sigmoid + activation for binary classification. + + Attributes: + X_train (array-like): Training feature data. + X_val (array-like): Validation feature data. + y_train (array-like): Training target data. + y_val (array-like): Validation target data. + + Methods: + _score(**params): Builds, compiles, and trains the MLP model using the + specified parameters for the hidden layer, and returns the validation + accuracy. + """ + def __init__(self, X_train, X_val, y_train, y_val): super().__init__() diff --git a/examples/v5_API_example/experiments/sklearn.py b/examples/v5_API_example/experiments/sklearn.py index d925167f..b1f92f04 100644 --- a/examples/v5_API_example/experiments/sklearn.py +++ b/examples/v5_API_example/experiments/sklearn.py @@ -6,6 +6,21 @@ class SklearnExperiment(BaseExperiment): + """ + Initializes the SklearnExperiment with the given estimator, data, and cross-validation settings. + + Parameters + ---------- + estimator : object + The machine learning estimator to be used for the experiment. + X : array-like + The input data for training the model. + y : array-like + The target values corresponding to the input data. + cv : int, optional + The number of cross-validation folds (default is 4). + """ + def __init__(self, estimator, X, y, cv=4): super().__init__() @@ -21,6 +36,23 @@ def _score(self, **params): class GradientBoostingExperiment(BaseExperiment): + """ + A class for conducting experiments with Gradient Boosting Regressor using cross-validation. + + This class inherits from BaseExperiment and allows users to perform experiments + with the GradientBoostingRegressor from sklearn. Users can specify the input + features, target values, and the number of cross-validation folds. + + Attributes: + estimator (type): The regression model to be used, default is GradientBoostingRegressor. + X (array-like): The input features for the model. + y (array-like): The target values for the model. + cv (int): The number of cross-validation folds. + + Methods: + _score(**params): Evaluates the model using cross-validation and returns the mean score. + """ + def __init__(self, X, y, cv=4): super().__init__() diff --git a/examples/v5_API_example/experiments/test_function.py b/examples/v5_API_example/experiments/test_function.py index 9260bfc4..18acb591 100644 --- a/examples/v5_API_example/experiments/test_function.py +++ b/examples/v5_API_example/experiments/test_function.py @@ -4,6 +4,23 @@ class SphereFunction(BaseExperiment): + """ + A class representing a Sphere function experiment. + + This class inherits from BaseExperiment and implements a simple + Sphere function, which is a common test function for optimization + algorithms. The function is defined as the sum of the squares of + its input parameters plus a constant. + + Attributes: + const (float): A constant added to the function's result. + n_dim (int): The number of dimensions for the input parameters. + + Methods: + _score(**params): Computes the Sphere function value for the + given parameters. + """ + def __init__(self, const, n_dim=2): super().__init__() @@ -15,6 +32,18 @@ def _score(self, **params): class AckleyFunction(BaseExperiment): + """ + A class representing the Ackley function, used as a benchmark for optimization algorithms. + + Attributes: + A (float): A constant used in the calculation of the Ackley function. + + Methods: + _score(**params): Computes the Ackley function value for given parameters 'x0' and 'x1'. + + The Ackley function is a non-convex function used to test optimization algorithms. + """ + def __init__(self, A): super().__init__() self.A = A diff --git a/examples/v5_API_example/optimizer.py b/examples/v5_API_example/optimizer.py index 8660599f..8f4ecb90 100644 --- a/examples/v5_API_example/optimizer.py +++ b/examples/v5_API_example/optimizer.py @@ -35,6 +35,7 @@ optimizer1.add_search(experiment, search_space) optimizer2.add_search(experiment, search_space) +# not sure about this way of combining optimizers. Might not be intuitive what the plus means. hyper = optimizer1 + optimizer2 hyper.run(max_time=5) diff --git a/examples/v5_API_example/search_space_optional.py b/examples/v5_API_example/search_space_optional.py index ef7271a9..2a17dd3d 100644 --- a/examples/v5_API_example/search_space_optional.py +++ b/examples/v5_API_example/search_space_optional.py @@ -2,6 +2,15 @@ from hyperactive import BaseSearchSpace +""" +Optional: + +It might make sense to use a class instead of a dictionary for the search-space. +The search-space can have specifc properties, that can be computed from the params (previously called search-space-dictionary). +The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. +""" + + class SearchSpace: search_space: dict = None From b5d6e7c5a797db82bb6fa0b0505f399e72911a84 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 18 Jan 2025 09:58:03 +0100 Subject: [PATCH 07/62] fix import path --- examples/v5_API_example/experiments/keras.py | 2 +- examples/v5_API_example/experiments/sklearn.py | 2 +- examples/v5_API_example/experiments/test_function.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/v5_API_example/experiments/keras.py b/examples/v5_API_example/experiments/keras.py index dce2441b..a7cc74fd 100644 --- a/examples/v5_API_example/experiments/keras.py +++ b/examples/v5_API_example/experiments/keras.py @@ -2,7 +2,7 @@ from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split -from hyperactive import BaseExperiment +from hyperactive.base import BaseExperiment X, y = make_classification(n_samples=1000, n_features=20, random_state=42) diff --git a/examples/v5_API_example/experiments/sklearn.py b/examples/v5_API_example/experiments/sklearn.py index b1f92f04..fe231cda 100644 --- a/examples/v5_API_example/experiments/sklearn.py +++ b/examples/v5_API_example/experiments/sklearn.py @@ -2,7 +2,7 @@ from sklearn.ensemble import GradientBoostingRegressor -from hyperactive import BaseExperiment +from hyperactive.base import BaseExperiment class SklearnExperiment(BaseExperiment): diff --git a/examples/v5_API_example/experiments/test_function.py b/examples/v5_API_example/experiments/test_function.py index 18acb591..22545ddb 100644 --- a/examples/v5_API_example/experiments/test_function.py +++ b/examples/v5_API_example/experiments/test_function.py @@ -1,6 +1,6 @@ import numpy as np -from hyperactive import BaseExperiment +from hyperactive.base import BaseExperiment class SphereFunction(BaseExperiment): From ca0379a221a58c31cbba0ba7df117296481cb594 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 18 Jan 2025 09:58:58 +0100 Subject: [PATCH 08/62] move search-space class --- examples/v5_API_example/optimizer.py | 3 ++- .../hyperactive/base/_search_space_optional.py | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename examples/v5_API_example/search_space_optional.py => src/hyperactive/base/_search_space_optional.py (100%) diff --git a/examples/v5_API_example/optimizer.py b/examples/v5_API_example/optimizer.py index 8f4ecb90..aa354a46 100644 --- a/examples/v5_API_example/optimizer.py +++ b/examples/v5_API_example/optimizer.py @@ -2,13 +2,14 @@ from sklearn.datasets import load_diabetes from sklearn.tree import DecisionTreeRegressor + +from hyperactive.base.search_space_optional import SearchSpace from hyperactive.optimizers import ( HillClimbingOptimizer, RandomRestartHillClimbingOptimizer, ) from .experiments.test_function import SklearnExperiment -from .search_space_optional import SearchSpace data = load_diabetes() diff --git a/examples/v5_API_example/search_space_optional.py b/src/hyperactive/base/_search_space_optional.py similarity index 100% rename from examples/v5_API_example/search_space_optional.py rename to src/hyperactive/base/_search_space_optional.py From e6da3ebfe038778729b72779633388753cc37e01 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 26 Jan 2025 08:33:16 +0100 Subject: [PATCH 09/62] put v4 API into separate dir --- src_old/hyperactive/__init__.py | 16 ++ src_old/hyperactive/base/__init__.py | 7 + src_old/hyperactive/base/_experiment.py | 71 +++++ src_old/hyperactive/base/_optimizer.py | 33 +++ .../hyperactive/base/search_space/__init__.py | 3 + .../base/search_space/_properties.py | 28 ++ .../base/search_space/_search_space.py | 107 ++++++++ src_old/hyperactive/distribution.py | 45 +++ src_old/hyperactive/hyperactive.py | 259 ++++++++++++++++++ src_old/hyperactive/integrations/__init__.py | 6 + .../integrations/sklearn/__init__.py | 6 + .../integrations/sklearn/best_estimator.py | 58 ++++ .../integrations/sklearn/checks.py | 10 + .../sklearn/hyperactive_search_cv.py | 172 ++++++++++++ .../sklearn/objective_function_adapter.py | 41 +++ .../sklearn/sklearn_cv_experiment.py | 59 ++++ .../hyperactive/integrations/sklearn/utils.py | 38 +++ .../integrations/sktime/__init__.py | 6 + .../hyperactive/integrations/sktime/main.py | 8 + src_old/hyperactive/optimizers/__init__.py | 55 ++++ src_old/hyperactive/optimizers/constraint.py | 22 ++ src_old/hyperactive/optimizers/dictionary.py | 17 ++ .../optimizers/hyper_gradient_conv.py | 131 +++++++++ .../hyperactive/optimizers/hyper_optimizer.py | 179 ++++++++++++ .../optimizers/objective_function.py | 51 ++++ .../optimizers/optimizer_attributes.py | 32 +++ src_old/hyperactive/optimizers/optimizers.py | 170 ++++++++++++ .../optimizers/strategies/__init__.py | 11 + .../custom_optimization_strategy.py | 22 ++ .../strategies/optimization_strategy.py | 127 +++++++++ .../strategies/optimizer_attributes.py | 64 +++++ src_old/hyperactive/print_results.py | 169 ++++++++++++ src_old/hyperactive/process.py | 36 +++ src_old/hyperactive/results.py | 88 ++++++ src_old/hyperactive/run_search.py | 57 ++++ src_old/hyperactive/search_space.py | 134 +++++++++ 36 files changed, 2338 insertions(+) create mode 100644 src_old/hyperactive/__init__.py create mode 100644 src_old/hyperactive/base/__init__.py create mode 100644 src_old/hyperactive/base/_experiment.py create mode 100644 src_old/hyperactive/base/_optimizer.py create mode 100644 src_old/hyperactive/base/search_space/__init__.py create mode 100644 src_old/hyperactive/base/search_space/_properties.py create mode 100644 src_old/hyperactive/base/search_space/_search_space.py create mode 100644 src_old/hyperactive/distribution.py create mode 100644 src_old/hyperactive/hyperactive.py create mode 100644 src_old/hyperactive/integrations/__init__.py create mode 100644 src_old/hyperactive/integrations/sklearn/__init__.py create mode 100644 src_old/hyperactive/integrations/sklearn/best_estimator.py create mode 100644 src_old/hyperactive/integrations/sklearn/checks.py create mode 100644 src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py create mode 100644 src_old/hyperactive/integrations/sklearn/objective_function_adapter.py create mode 100644 src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py create mode 100644 src_old/hyperactive/integrations/sklearn/utils.py create mode 100644 src_old/hyperactive/integrations/sktime/__init__.py create mode 100644 src_old/hyperactive/integrations/sktime/main.py create mode 100644 src_old/hyperactive/optimizers/__init__.py create mode 100644 src_old/hyperactive/optimizers/constraint.py create mode 100644 src_old/hyperactive/optimizers/dictionary.py create mode 100644 src_old/hyperactive/optimizers/hyper_gradient_conv.py create mode 100644 src_old/hyperactive/optimizers/hyper_optimizer.py create mode 100644 src_old/hyperactive/optimizers/objective_function.py create mode 100644 src_old/hyperactive/optimizers/optimizer_attributes.py create mode 100644 src_old/hyperactive/optimizers/optimizers.py create mode 100644 src_old/hyperactive/optimizers/strategies/__init__.py create mode 100644 src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py create mode 100644 src_old/hyperactive/optimizers/strategies/optimization_strategy.py create mode 100644 src_old/hyperactive/optimizers/strategies/optimizer_attributes.py create mode 100644 src_old/hyperactive/print_results.py create mode 100644 src_old/hyperactive/process.py create mode 100644 src_old/hyperactive/results.py create mode 100644 src_old/hyperactive/run_search.py create mode 100644 src_old/hyperactive/search_space.py diff --git a/src_old/hyperactive/__init__.py b/src_old/hyperactive/__init__.py new file mode 100644 index 00000000..bc82b676 --- /dev/null +++ b/src_old/hyperactive/__init__.py @@ -0,0 +1,16 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import importlib.metadata + +__version__ = importlib.metadata.version("hyperactive") +__license__ = "MIT" + + +from .hyperactive import Hyperactive + + +__all__ = [ + "Hyperactive", +] diff --git a/src_old/hyperactive/base/__init__.py b/src_old/hyperactive/base/__init__.py new file mode 100644 index 00000000..e2c2df68 --- /dev/null +++ b/src_old/hyperactive/base/__init__.py @@ -0,0 +1,7 @@ +"""Base classes for optimizers and experiments.""" + +from ._experiment import BaseExperiment +from ._optimizer import BaseOptimizer +from .search_space import SearchSpace + +__all__ = ["BaseExperiment", "BaseOptimizer", "SearchSpace"] diff --git a/src_old/hyperactive/base/_experiment.py b/src_old/hyperactive/base/_experiment.py new file mode 100644 index 00000000..ad3a5999 --- /dev/null +++ b/src_old/hyperactive/base/_experiment.py @@ -0,0 +1,71 @@ +"""Base class for experiment.""" + +from skbase.base import BaseObject + + +class BaseExperiment(BaseObject): + """Base class for experiment.""" + + def __init__(self): + super().__init__() + + def __call__(self, **kwargs): + """Score parameters, with kwargs call.""" + return self.score(kwargs) + + def paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return self._paramnames() + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + raise NotImplementedError + + def score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + paramnames = self.paramnames() + if not set(paramnames) == set(params.keys()): + raise ValueError("Parameters do not match.") + return self._score(**params) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + raise NotImplementedError diff --git a/src_old/hyperactive/base/_optimizer.py b/src_old/hyperactive/base/_optimizer.py new file mode 100644 index 00000000..55bf7d1a --- /dev/null +++ b/src_old/hyperactive/base/_optimizer.py @@ -0,0 +1,33 @@ +"""Base class for optimizer.""" + +from skbase.base import BaseObject + + +class BaseOptimizer(BaseObject): + """Base class for optimizer.""" + + def __init__(self): + super().__init__() + + def add_search(self, experiment, search_config: dict): + """Add a new optimization search process with specified parameters. + + Parameters + ---------- + experiment : BaseExperiment + The experiment to optimize parameters for. + search_config : dict with str keys + The search configuration dictionary. + """ + self.experiment = experiment + self.search_config = search_config + + def run(self, max_time=None): + """Run the optimization search process. + + Parameters + ---------- + max_time : float + The maximum time used for the optimization process. + """ + raise NotImplementedError diff --git a/src_old/hyperactive/base/search_space/__init__.py b/src_old/hyperactive/base/search_space/__init__.py new file mode 100644 index 00000000..b341faf0 --- /dev/null +++ b/src_old/hyperactive/base/search_space/__init__.py @@ -0,0 +1,3 @@ +from ._search_space import SearchSpace + +__all__ = ["SearchSpace"] diff --git a/src_old/hyperactive/base/search_space/_properties.py b/src_old/hyperactive/base/search_space/_properties.py new file mode 100644 index 00000000..aa1cf543 --- /dev/null +++ b/src_old/hyperactive/base/search_space/_properties.py @@ -0,0 +1,28 @@ +import numpy as np + + +def n_dim(search_space): + return len(search_space) + + +def dim_names(search_space): + return list(search_space.keys()) + + +def position_space(search_space): + return { + key: np.array(range(len(search_space[key]))) + for key in search_space.keys() + } + + +def calculate_properties(func): + def wrapper(self, *args, **kwargs): + func(self, *args, **kwargs) + + self.n_dim = n_dim(self._search_space) + self.dim_names = dim_names(self._search_space) + self.position_space = position_space(self._search_space) + print(" ---> search-space updated!") + + return wrapper diff --git a/src_old/hyperactive/base/search_space/_search_space.py b/src_old/hyperactive/base/search_space/_search_space.py new file mode 100644 index 00000000..fadb6c6f --- /dev/null +++ b/src_old/hyperactive/base/search_space/_search_space.py @@ -0,0 +1,107 @@ +""" +Optional: + +It might make sense to use a class instead of a dictionary for the search-space. +The search-space can have specifc properties, that can be computed from the params (previously called search-space-dictionary). +The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. +""" + +from collections.abc import MutableMapping + +from ._properties import calculate_properties + + +class SearchSpaceDictLike(MutableMapping): + _search_space: dict = None + + def __init__(self, **param_space): + self._search_space = dict(**param_space) + + def __getitem__(self, key): + return self._search_space[key] + + def __setitem__(self, key, value): + self._search_space[key] = value + + def __delitem__(self, key): + del self._search_space[key] + + def __iter__(self): + return iter(self._search_space) + + def __len__(self): + return len(self._search_space) + + def __repr__(self): + return f"{self.__class__.__name__}({self._search_space})" + + def __call__(self): + return self._search_space + + def keys(self): + return self._search_space.keys() + + def values(self): + return self._search_space.values() + + def items(self): + return self._search_space.items() + + +class SearchSpace(SearchSpaceDictLike): + + @calculate_properties + def __init__(self, **param_space): + super().__init__(**param_space) + + for key, value in param_space.items(): + setattr(self, key, value) + + @calculate_properties + def __setitem__(self, key, value): + SearchSpaceDictLike.__setitem__(self, key, value) + + @calculate_properties + def __delitem__(self, key): + SearchSpaceDictLike.__delitem__(self, key) + + def print(self, indent=2, max_list_length=5): + """ + Prints the dictionary in a readable format. + + Args: + indent (int): The number of spaces to indent nested structures. + max_list_length (int): The maximum number of items to display from long lists. + """ + + def format_value(value, level=0): + prefix = " " * (level * indent) + if isinstance(value, list): + if len(value) > max_list_length: + # Truncate long lists for readability + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value[:max_list_length] + ) + result += f"{prefix}{' ' * indent}... ({len(value) - max_list_length} more items)\n" + result += f"{prefix}]" + else: + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value + ) + result += f"{prefix}]" + elif isinstance(value, dict): + # Format nested dictionaries + result = "{\n" + for k, v in value.items(): + result += f"{prefix}{' ' * indent}{repr(k)}: {format_value(v, level + 1)},\n" + result += f"{prefix}}}" + else: + result = repr(value) + return result + + for key, value in self.items(): + print(f"{key}: {format_value(value)}") diff --git a/src_old/hyperactive/distribution.py b/src_old/hyperactive/distribution.py new file mode 100644 index 00000000..eea37214 --- /dev/null +++ b/src_old/hyperactive/distribution.py @@ -0,0 +1,45 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from sys import platform +from tqdm import tqdm + +if platform.startswith("linux"): + initializer = tqdm.set_lock + initargs = (tqdm.get_lock(),) +else: + initializer = None + initargs = () + + +def single_process(process_func, process_infos): + return [process_func(*info) for info in process_infos] + + +def multiprocessing_wrapper(process_func, process_infos, n_processes): + import multiprocessing as mp + + with mp.Pool( + n_processes, initializer=initializer, initargs=initargs + ) as pool: + return pool.map(process_func, process_infos) + + +def pathos_wrapper(process_func, search_processes_paras, n_processes): + import pathos.multiprocessing as pmp + + with pmp.Pool( + n_processes, initializer=initializer, initargs=initargs + ) as pool: + return pool.map(process_func, search_processes_paras) + + +def joblib_wrapper(process_func, search_processes_paras, n_processes): + from joblib import Parallel, delayed + + jobs = [ + delayed(process_func)(*info_dict) + for info_dict in search_processes_paras + ] + return Parallel(n_jobs=n_processes)(jobs) diff --git a/src_old/hyperactive/hyperactive.py b/src_old/hyperactive/hyperactive.py new file mode 100644 index 00000000..18face95 --- /dev/null +++ b/src_old/hyperactive/hyperactive.py @@ -0,0 +1,259 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +import copy +import multiprocessing as mp +import pandas as pd + +from typing import Union, List, Dict, Type + +from .optimizers import RandomSearchOptimizer +from .run_search import run_search + +from .results import Results +from .print_results import PrintResults +from .search_space import SearchSpace + + +class Hyperactive: + """ + Initialize the Hyperactive class to manage optimization processes. + + Parameters: + - verbosity: List of verbosity levels (default: ["progress_bar", "print_results", "print_times"]) + - distribution: String indicating the distribution method (default: "multiprocessing") + - n_processes: Number of processes to run in parallel or "auto" to determine automatically (default: "auto") + + Methods: + - add_search: Add a new optimization search process with specified parameters + - run: Execute the optimization searches + - best_para: Get the best parameters for a specific search + - best_score: Get the best score for a specific search + - search_data: Get the search data for a specific search + """ + + def __init__( + self, + verbosity: list = ["progress_bar", "print_results", "print_times"], + distribution: str = "multiprocessing", + n_processes: Union[str, int] = "auto", + ): + super().__init__() + if verbosity is False: + verbosity = [] + + self.verbosity = verbosity + self.distribution = distribution + self.n_processes = n_processes + + self.opt_pros = {} + + def _create_shared_memory(self): + _bundle_opt_processes = {} + + for opt_pros in self.opt_pros.values(): + if opt_pros.memory != "share": + continue + name = opt_pros.objective_function.__name__ + + _bundle_opt_processes.setdefault(name, []).append(opt_pros) + + for opt_pros_l in _bundle_opt_processes.values(): + # Check if the lengths of the search spaces of all optimizers in the list are the same. + if ( + len(set(len(opt_pros.s_space()) for opt_pros in opt_pros_l)) + == 1 + ): + manager = mp.Manager() # get new manager.dict + shared_memory = manager.dict() + for opt_pros in opt_pros_l: + opt_pros.memory = shared_memory + else: + for opt_pros in opt_pros_l: + opt_pros.memory = opt_pros_l[ + 0 + ].memory # get same manager.dict + + @staticmethod + def _default_opt(optimizer): + if isinstance(optimizer, str): + if optimizer == "default": + optimizer = RandomSearchOptimizer() + return copy.deepcopy(optimizer) + + @staticmethod + def _default_search_id(search_id, objective_function): + if not search_id: + search_id = objective_function.__name__ + return search_id + + @staticmethod + def check_list(search_space): + for key in search_space.keys(): + search_dim = search_space[key] + + error_msg = "Value in '{}' of search space dictionary must be of type list".format( + key + ) + if not isinstance(search_dim, list): + print("Warning", error_msg) + # raise ValueError(error_msg) + + def add_search( + self, + objective_function: callable, + search_space: Dict[str, list], + n_iter: int, + search_id=None, + optimizer: Union[str, Type[RandomSearchOptimizer]] = "default", + n_jobs: int = 1, + initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, + constraints: List[callable] = None, + pass_through: Dict = None, + callbacks: Dict[str, callable] = None, + catch: Dict = None, + max_score: float = None, + early_stopping: Dict = None, + random_state: int = None, + memory: Union[str, bool] = "share", + memory_warm_start: pd.DataFrame = None, + ): + """ + Add a new optimization search process with specified parameters. + + Parameters: + - objective_function: The objective function to optimize. + - search_space: Dictionary defining the search space for optimization. + - n_iter: Number of iterations for the optimization process. + - search_id: Identifier for the search process (default: None). + - optimizer: The optimizer to use for the search process (default: "default"). + - n_jobs: Number of parallel jobs to run (default: 1). + - initialize: Dictionary specifying initialization parameters (default: {"grid": 4, "random": 2, "vertices": 4}). + - constraints: List of constraint functions (default: None). + - pass_through: Dictionary of additional parameters to pass through (default: None). + - callbacks: Dictionary of callback functions (default: None). + - catch: Dictionary of exceptions to catch during optimization (default: None). + - max_score: Maximum score to achieve (default: None). + - early_stopping: Dictionary specifying early stopping criteria (default: None). + - random_state: Seed for random number generation (default: None). + - memory: Option to share memory between processes (default: "share"). + - memory_warm_start: DataFrame containing warm start memory (default: None). + """ + + self.check_list(search_space) + + constraints = constraints or [] + pass_through = pass_through or {} + callbacks = callbacks or {} + catch = catch or {} + early_stopping = early_stopping or {} + + optimizer = self._default_opt(optimizer) + search_id = self._default_search_id(search_id, objective_function) + s_space = SearchSpace(search_space) + + optimizer.setup_search( + objective_function=objective_function, + s_space=s_space, + n_iter=n_iter, + initialize=initialize, + constraints=constraints, + pass_through=pass_through, + callbacks=callbacks, + catch=catch, + max_score=max_score, + early_stopping=early_stopping, + random_state=random_state, + memory=memory, + memory_warm_start=memory_warm_start, + verbosity=self.verbosity, + ) + + n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs + + for _ in range(n_jobs): + nth_process = len(self.opt_pros) + self.opt_pros[nth_process] = optimizer + + def _print_info(self): + print_res = PrintResults(self.opt_pros, self.verbosity) + + if self.verbosity: + for _ in range(len(self.opt_pros)): + print("") + + for results in self.results_list: + nth_process = results["nth_process"] + print_res.print_process(results, nth_process) + + def run(self, max_time: float = None): + """ + Run the optimization process with an optional maximum time limit. + + Args: + max_time (float, optional): Maximum time limit for the optimization process. Defaults to None. + """ + + self._create_shared_memory() + + for opt in self.opt_pros.values(): + opt.max_time = max_time + + self.results_list = run_search( + self.opt_pros, self.distribution, self.n_processes + ) + + self.results_ = Results(self.results_list, self.opt_pros) + + self._print_info() + + def best_para(self, id_): + """ + Retrieve the best parameters for a specific ID from the results. + + Parameters: + - id_ (int): The ID of the parameters to retrieve. + + Returns: + - Union[Dict[str, Union[int, float]], None]: The best parameters for the specified ID if found, otherwise None. + + Raises: + - ValueError: If the objective function name is not recognized. + """ + + return self.results_.best_para(id_) + + def best_score(self, id_): + """ + Return the best score for a specific ID from the results. + + Parameters: + - id_ (int): The ID for which the best score is requested. + """ + + return self.results_.best_score(id_) + + def search_data(self, id_, times=False): + """ + Retrieve search data for a specific ID from the results. Optionally exclude evaluation and iteration times if 'times' is set to False. + + Parameters: + - id_ (int): The ID of the search data to retrieve. + - times (bool, optional): Whether to exclude evaluation and iteration times. Defaults to False. + + Returns: + - pd.DataFrame: The search data for the specified ID. + """ + + search_data_ = self.results_.search_data(id_) + + if times == False: + search_data_.drop( + labels=["eval_times", "iter_times"], + axis=1, + inplace=True, + errors="ignore", + ) + return search_data_ diff --git a/src_old/hyperactive/integrations/__init__.py b/src_old/hyperactive/integrations/__init__.py new file mode 100644 index 00000000..7d540ced --- /dev/null +++ b/src_old/hyperactive/integrations/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .sklearn import HyperactiveSearchCV diff --git a/src_old/hyperactive/integrations/sklearn/__init__.py b/src_old/hyperactive/integrations/sklearn/__init__.py new file mode 100644 index 00000000..b5e193f9 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .hyperactive_search_cv import HyperactiveSearchCV diff --git a/src_old/hyperactive/integrations/sklearn/best_estimator.py b/src_old/hyperactive/integrations/sklearn/best_estimator.py new file mode 100644 index 00000000..def5f828 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/best_estimator.py @@ -0,0 +1,58 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from sklearn.utils.metaestimators import available_if +from sklearn.utils.deprecation import _deprecate_Xt_in_inverse_transform +from sklearn.exceptions import NotFittedError +from sklearn.utils.validation import check_is_fitted + +from .utils import _estimator_has + + +# NOTE Implementations of following methods from: +# https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/model_selection/_search.py +# Tag: 1.5.1 +class BestEstimator: + + @available_if(_estimator_has("score_samples")) + def score_samples(self, X): + check_is_fitted(self) + return self.best_estimator_.score_samples(X) + + @available_if(_estimator_has("predict")) + def predict(self, X): + check_is_fitted(self) + return self.best_estimator_.predict(X) + + @available_if(_estimator_has("predict_proba")) + def predict_proba(self, X): + check_is_fitted(self) + return self.best_estimator_.predict_proba(X) + + @available_if(_estimator_has("predict_log_proba")) + def predict_log_proba(self, X): + check_is_fitted(self) + return self.best_estimator_.predict_log_proba(X) + + @available_if(_estimator_has("decision_function")) + def decision_function(self, X): + check_is_fitted(self) + return self.best_estimator_.decision_function(X) + + @available_if(_estimator_has("transform")) + def transform(self, X): + check_is_fitted(self) + return self.best_estimator_.transform(X) + + @available_if(_estimator_has("inverse_transform")) + def inverse_transform(self, X=None, Xt=None): + X = _deprecate_Xt_in_inverse_transform(X, Xt) + check_is_fitted(self) + return self.best_estimator_.inverse_transform(X) + + @property + def classes_(self): + _estimator_has("classes_")(self) + return self.best_estimator_.classes_ diff --git a/src_old/hyperactive/integrations/sklearn/checks.py b/src_old/hyperactive/integrations/sklearn/checks.py new file mode 100644 index 00000000..2d107a0a --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/checks.py @@ -0,0 +1,10 @@ +class Checks: + _fit_successful = False + + def verify_fit(function): + def wrapper(self, X, y): + out = function(self, X, y) + self._fit_successful = True + return out + + return wrapper diff --git a/src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py b/src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py new file mode 100644 index 00000000..a7db6550 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py @@ -0,0 +1,172 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from collections.abc import Iterable, Callable +from typing import Union, Dict, Type + +from sklearn.base import BaseEstimator, clone +from sklearn.metrics import check_scoring +from sklearn.utils.validation import indexable, _check_method_params + +from sklearn.base import BaseEstimator as SklearnBaseEstimator +from sklearn.model_selection import BaseCrossValidator + +from hyperactive import Hyperactive + +from .objective_function_adapter import ObjectiveFunctionAdapter +from .best_estimator import BestEstimator as _BestEstimator_ +from .checks import Checks +from ...optimizers import RandomSearchOptimizer +from hyperactive.integrations.sklearn.sklearn_cv_experiment import SklearnCvExperiment + + +class HyperactiveSearchCV(BaseEstimator, _BestEstimator_, Checks): + """ + HyperactiveSearchCV class for hyperparameter tuning using cross-validation with sklearn estimators. + + Parameters: + - estimator: SklearnBaseEstimator + The estimator to be tuned. + - params_config: Dict[str, list] + Dictionary containing the hyperparameter search space. + - optimizer: Union[str, Type[RandomSearchOptimizer]], optional + The optimizer to be used for hyperparameter search, default is "default". + - n_iter: int, optional + Number of parameter settings that are sampled, default is 100. + - scoring: Callable | str | None, optional + Scoring method to evaluate the predictions on the test set. + - n_jobs: int, optional + Number of jobs to run in parallel, default is 1. + - random_state: int | None, optional + Random seed for reproducibility. + - refit: bool, optional + Refit the best estimator with the entire dataset, default is True. + - cv: int | "BaseCrossValidator" | Iterable | None, optional + Determines the cross-validation splitting strategy. + + Methods: + - fit(X, y, **fit_params) + Fit the estimator and tune hyperparameters. + - score(X, y, **params) + Return the score of the best estimator on the input data. + """ + + _required_parameters = ["estimator", "optimizer", "params_config"] + + def __init__( + self, + estimator: "SklearnBaseEstimator", + params_config: Dict[str, list], + optimizer: Union[str, Type[RandomSearchOptimizer]] = "default", + n_iter: int = 100, + *, + scoring: Union[Callable, str, None] = None, + n_jobs: int = 1, + random_state: Union[int, None] = None, + refit: bool = True, + cv=None, + ): + super().__init__() + + self.estimator = estimator + self.params_config = params_config + self.optimizer = optimizer + self.n_iter = n_iter + self.scoring = scoring + self.n_jobs = n_jobs + self.random_state = random_state + self.refit = refit + self.cv = cv + + def _refit(self, X, y=None, **fit_params): + self.best_estimator_ = clone(self.estimator).set_params( + **clone(self.best_params_, safe=False) + ) + + self.best_estimator_.fit(X, y, **fit_params) + return self + + def _check_data(self, X, y): + X, y = indexable(X, y) + if hasattr(self, "_validate_data"): + validate_data = self._validate_data + else: + from sklearn.utils.validation import validate_data + + return validate_data(X, y) + + @Checks.verify_fit + def fit(self, X, y, **fit_params): + """ + Fit the estimator using the provided training data. + + Parameters: + - X: array-like or sparse matrix, shape (n_samples, n_features) + The training input samples. + - y: array-like, shape (n_samples,) or (n_samples, n_outputs) + The target values. + - **fit_params: dict of string -> object + Additional fit parameters. + + Returns: + - self: object + Returns the instance itself. + """ + + X, y = self._check_data(X, y) + + fit_params = _check_method_params(X, params=fit_params) + self.scorer_ = check_scoring(self.estimator, scoring=self.scoring) + + experiment = SklearnCvExperiment( + estimator=self.estimator, + scoring=self.scorer_, + cv=self.cv, + X=X, + y=y, + ) + objective_function = experiment.score + + hyper = Hyperactive(verbosity=False) + hyper.add_search( + objective_function, + search_space=self.params_config, + optimizer=self.optimizer, + n_iter=self.n_iter, + n_jobs=self.n_jobs, + random_state=self.random_state, + ) + hyper.run() + + self.best_params_ = hyper.best_para(objective_function) + self.best_score_ = hyper.best_score(objective_function) + self.search_data_ = hyper.search_data(objective_function) + + if self.refit: + self._refit(X, y, **fit_params) + + return self + + def score(self, X, y=None, **params): + """ + Calculate the score of the best estimator on the input data. + + Parameters: + - X: array-like or sparse matrix of shape (n_samples, n_features) + The input samples. + - y: array-like of shape (n_samples,), default=None + The target values. + - **params: dict + Additional parameters to be passed to the scoring function. + + Returns: + - float + The score of the best estimator on the input data. + """ + + return self.scorer_(self.best_estimator_, X, y, **params) + + @property + def fit_successful(self): + self._fit_successful diff --git a/src_old/hyperactive/integrations/sklearn/objective_function_adapter.py b/src_old/hyperactive/integrations/sklearn/objective_function_adapter.py new file mode 100644 index 00000000..9f8fde4a --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/objective_function_adapter.py @@ -0,0 +1,41 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from sklearn import clone +from sklearn.model_selection import cross_validate +from sklearn.utils.validation import _num_samples + + +class ObjectiveFunctionAdapter: + def __init__(self, estimator) -> None: + self.estimator = estimator + + def add_dataset(self, X, y): + self.X = X + self.y = y + + def add_validation(self, scoring, cv): + self.scoring = scoring + self.cv = cv + + def objective_function(self, params): + + estimator = clone(self.estimator) + estimator.set_params(**params) + + cv_results = cross_validate( + estimator, + self.X, + self.y, + cv=self.cv, + ) + + add_info_d = { + "score_time": cv_results["score_time"], + "fit_time": cv_results["fit_time"], + "n_test_samples": _num_samples(self.X), + } + + return cv_results["test_score"].mean(), add_info_d diff --git a/src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py b/src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py new file mode 100644 index 00000000..b77aaf7d --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py @@ -0,0 +1,59 @@ +"""Experiment adapter for sklearn cross-validation experiments.""" + +from sklearn import clone +from sklearn.model_selection import cross_validate +from sklearn.utils.validation import _num_samples + +from hyperactive.base import BaseExperiment + +class SklearnCvExperiment(BaseExperiment): + + def __init__(self, estimator, scoring, cv, X, y): + self.estimator = estimator + self.X = X + self.y = y + self.scoring = scoring + self.cv = cv + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return list(self.estimator.get_params().keys()) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + estimator = clone(self.estimator) + estimator.set_params(**params) + + cv_results = cross_validate( + self.estimator, + self.X, + self.y, + cv=self.cv, + ) + + add_info_d = { + "score_time": cv_results["score_time"], + "fit_time": cv_results["fit_time"], + "n_test_samples": _num_samples(self.X), + } + + return cv_results["test_score"].mean(), add_info_d diff --git a/src_old/hyperactive/integrations/sklearn/utils.py b/src_old/hyperactive/integrations/sklearn/utils.py new file mode 100644 index 00000000..6a25cb19 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/utils.py @@ -0,0 +1,38 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from sklearn.utils.validation import ( + indexable, + _check_method_params, + check_is_fitted, +) + +# NOTE Implementations of following methods from: +# https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/model_selection/_search.py +# Tag: 1.5.1 + + +def _check_refit(search_cv, attr): + if not search_cv.refit: + raise AttributeError( + f"This {type(search_cv).__name__} instance was initialized with " + f"`refit=False`. {attr} is available only after refitting on the best " + "parameters. You can refit an estimator manually using the " + "`best_params_` attribute" + ) + + +def _estimator_has(attr): + def check(self): + _check_refit(self, attr) + if hasattr(self, "best_estimator_"): + # raise an AttributeError if `attr` does not exist + getattr(self.best_estimator_, attr) + return True + # raise an AttributeError if `attr` does not exist + getattr(self.estimator, attr) + return True + + return check diff --git a/src_old/hyperactive/integrations/sktime/__init__.py b/src_old/hyperactive/integrations/sktime/__init__.py new file mode 100644 index 00000000..09bdbd71 --- /dev/null +++ b/src_old/hyperactive/integrations/sktime/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .main import HyperactiveSearchCV diff --git a/src_old/hyperactive/integrations/sktime/main.py b/src_old/hyperactive/integrations/sktime/main.py new file mode 100644 index 00000000..f0d7c5db --- /dev/null +++ b/src_old/hyperactive/integrations/sktime/main.py @@ -0,0 +1,8 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class HyperactiveSearchCV: + def __init__(self) -> None: + pass diff --git a/src_old/hyperactive/optimizers/__init__.py b/src_old/hyperactive/optimizers/__init__.py new file mode 100644 index 00000000..c476012e --- /dev/null +++ b/src_old/hyperactive/optimizers/__init__.py @@ -0,0 +1,55 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .optimizers import ( + HillClimbingOptimizer, + StochasticHillClimbingOptimizer, + RepulsingHillClimbingOptimizer, + SimulatedAnnealingOptimizer, + DownhillSimplexOptimizer, + RandomSearchOptimizer, + GridSearchOptimizer, + RandomRestartHillClimbingOptimizer, + RandomAnnealingOptimizer, + PowellsMethod, + PatternSearch, + ParallelTemperingOptimizer, + ParticleSwarmOptimizer, + SpiralOptimization, + GeneticAlgorithmOptimizer, + EvolutionStrategyOptimizer, + DifferentialEvolutionOptimizer, + BayesianOptimizer, + LipschitzOptimizer, + DirectAlgorithm, + TreeStructuredParzenEstimators, + ForestOptimizer, +) + + +__all__ = [ + "HillClimbingOptimizer", + "StochasticHillClimbingOptimizer", + "RepulsingHillClimbingOptimizer", + "SimulatedAnnealingOptimizer", + "DownhillSimplexOptimizer", + "RandomSearchOptimizer", + "GridSearchOptimizer", + "RandomRestartHillClimbingOptimizer", + "RandomAnnealingOptimizer", + "PowellsMethod", + "PatternSearch", + "ParallelTemperingOptimizer", + "ParticleSwarmOptimizer", + "SpiralOptimization", + "GeneticAlgorithmOptimizer", + "EvolutionStrategyOptimizer", + "DifferentialEvolutionOptimizer", + "BayesianOptimizer", + "LipschitzOptimizer", + "DirectAlgorithm", + "TreeStructuredParzenEstimators", + "ForestOptimizer", +] diff --git a/src_old/hyperactive/optimizers/constraint.py b/src_old/hyperactive/optimizers/constraint.py new file mode 100644 index 00000000..e170970e --- /dev/null +++ b/src_old/hyperactive/optimizers/constraint.py @@ -0,0 +1,22 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +def gfo2hyper(search_space, para): + values_dict = {} + for _, key in enumerate(search_space.keys()): + pos_ = int(para[key]) + values_dict[key] = search_space[key][pos_] + + return values_dict + + +class Constraint: + def __init__(self, constraint, search_space): + self.constraint = constraint + self.search_space = search_space + + def __call__(self, para): + para = gfo2hyper(self.search_space, para) + return self.constraint(para) diff --git a/src_old/hyperactive/optimizers/dictionary.py b/src_old/hyperactive/optimizers/dictionary.py new file mode 100644 index 00000000..ca30e652 --- /dev/null +++ b/src_old/hyperactive/optimizers/dictionary.py @@ -0,0 +1,17 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class DictClass: + def __init__(self): + self.para_dict = {} + + def __getitem__(self, key): + return self.para_dict[key] + + def keys(self): + return self.para_dict.keys() + + def values(self): + return self.para_dict.values() diff --git a/src_old/hyperactive/optimizers/hyper_gradient_conv.py b/src_old/hyperactive/optimizers/hyper_gradient_conv.py new file mode 100644 index 00000000..d06d9aa8 --- /dev/null +++ b/src_old/hyperactive/optimizers/hyper_gradient_conv.py @@ -0,0 +1,131 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import numpy as np +import pandas as pd + + +class HyperGradientConv: + def __init__(self, s_space): + self.s_space = s_space + + def value2position(self, value: list) -> list: + return [ + np.abs(v - np.array(space_dim)).argmin() + for v, space_dim in zip(value, self.s_space.values_l) + ] + + def value2para(self, value: list) -> dict: + return {key: p for key, p in zip(self.s_space.dim_keys, value)} + + def para2value(self, para: dict) -> list: + return [para[para_name] for para_name in self.s_space.dim_keys] + + def position2value(self, position): + return [ + space_dim[pos] + for pos, space_dim in zip(position, self.s_space.values_l) + ] + + def para_func2str(self, para): + return { + dim_key: ( + para[dim_key].__name__ + if self.s_space.data_types[dim_key] != "number" + else para[dim_key] + ) + for dim_key in self.s_space.dim_keys + } + + def value_func2str(self, value): + try: + return value.__name__ + except: + return value + + def conv_para(self, para_hyper): + para_gfo = {} + for para in self.s_space.dim_keys: + value_hyper = para_hyper[para] + space_dim = list(self.s_space.func2str[para]) + + if self.s_space.data_types[para] == "number": + value_gfo = np.abs(value_hyper - np.array(space_dim)).argmin() + else: + value_hyper = self.value_func2str(value_hyper) + + if value_hyper in space_dim: + value_gfo = space_dim.index(value_hyper) + else: + raise ValueError( + f"'{value_hyper}' was not found in '{para}'" + ) + + para_gfo[para] = value_gfo + return para_gfo + + def conv_initialize(self, initialize): + if "warm_start" in initialize: + warm_start_l = initialize["warm_start"] + warm_start_gfo = [ + self.conv_para(warm_start) for warm_start in warm_start_l + ] + initialize["warm_start"] = warm_start_gfo + + return initialize + + def get_list_positions(self, list1_values, search_dim): + return [search_dim.index(value2) for value2 in list1_values] + + def values2positions(self, values, search_dim): + return np.array(search_dim).searchsorted(values) + + def positions2results(self, positions): + results_dict = {} + + for para_name in self.s_space.dim_keys: + values_list = self.s_space[para_name] + pos_ = positions[para_name].values + values_ = [values_list[idx] for idx in pos_] + results_dict[para_name] = values_ + + results = pd.DataFrame.from_dict(results_dict) + + diff_list = np.setdiff1d(positions.columns, results.columns) + results[diff_list] = positions[diff_list] + + return results + + def conv_memory_warm_start(self, results): + if results is None: + return results + + results.reset_index(inplace=True, drop=True) + + df_positions_dict = {} + for dim_key in self.s_space.dim_keys: + result_dim_values = list(results[dim_key].values) + search_dim = self.s_space.func2str[dim_key] + + if self.s_space.data_types[dim_key] == "object": + result_dim_values = [ + self.value_func2str(value) for value in result_dim_values + ] + + list1_positions = self.get_list_positions( + result_dim_values, search_dim + ) + else: + list1_positions = self.values2positions( + result_dim_values, search_dim + ) + + df_positions_dict[dim_key] = list1_positions + + results_new = pd.DataFrame(df_positions_dict) + + results_new["score"] = results["score"] + results_new.dropna(how="any", inplace=True) + + return results_new diff --git a/src_old/hyperactive/optimizers/hyper_optimizer.py b/src_old/hyperactive/optimizers/hyper_optimizer.py new file mode 100644 index 00000000..c7710023 --- /dev/null +++ b/src_old/hyperactive/optimizers/hyper_optimizer.py @@ -0,0 +1,179 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import numpy as np + +from .objective_function import ObjectiveFunction +from .hyper_gradient_conv import HyperGradientConv +from .optimizer_attributes import OptimizerAttributes +from .constraint import Constraint + + +class HyperOptimizer(OptimizerAttributes): + def __init__(self, **opt_params): + super().__init__() + self.opt_params = opt_params + + def setup_search( + self, + objective_function, + s_space, + n_iter, + initialize, + constraints, + pass_through, + callbacks, + catch, + max_score, + early_stopping, + random_state, + memory, + memory_warm_start, + verbosity, + ): + self.objective_function = objective_function + self.s_space = s_space + self.n_iter = n_iter + + self.initialize = initialize + self.constraints = constraints + self.pass_through = pass_through + self.callbacks = callbacks + self.catch = catch + self.max_score = max_score + self.early_stopping = early_stopping + self.random_state = random_state + self.memory = memory + self.memory_warm_start = memory_warm_start + self.verbosity = verbosity + + if "progress_bar" in self.verbosity: + self.verbosity = ["progress_bar"] + else: + self.verbosity = [] + + def convert_results2hyper(self): + self.eval_times = sum(self.gfo_optimizer.eval_times) + self.iter_times = sum(self.gfo_optimizer.iter_times) + + if self.gfo_optimizer.best_para is not None: + value = self.hg_conv.para2value(self.gfo_optimizer.best_para) + position = self.hg_conv.position2value(value) + best_para = self.hg_conv.value2para(position) + self.best_para = best_para + else: + self.best_para = None + + self.best_score = self.gfo_optimizer.best_score + self.positions = self.gfo_optimizer.search_data + self.search_data = self.hg_conv.positions2results(self.positions) + + results_dd = self.gfo_optimizer.search_data.drop_duplicates( + subset=self.s_space.dim_keys, keep="first" + ) + self.memory_values_df = results_dd[ + self.s_space.dim_keys + ["score"] + ].reset_index(drop=True) + + def _setup_process(self, nth_process): + self.nth_process = nth_process + + self.hg_conv = HyperGradientConv(self.s_space) + + initialize = self.hg_conv.conv_initialize(self.initialize) + search_space_positions = self.s_space.positions + + # conv warm start for smbo from values into positions + if "warm_start_smbo" in self.opt_params: + self.opt_params["warm_start_smbo"] = ( + self.hg_conv.conv_memory_warm_start( + self.opt_params["warm_start_smbo"] + ) + ) + + gfo_constraints = [ + Constraint(constraint, self.s_space) + for constraint in self.constraints + ] + + self.gfo_optimizer = self.optimizer_class( + search_space=search_space_positions, + initialize=initialize, + constraints=gfo_constraints, + random_state=self.random_state, + nth_process=nth_process, + **self.opt_params, + ) + + self.conv = self.gfo_optimizer.conv + + def search(self, nth_process, p_bar): + self._setup_process(nth_process) + + gfo_wrapper_model = ObjectiveFunction( + objective_function=self.objective_function, + optimizer=self.gfo_optimizer, + callbacks=self.callbacks, + catch=self.catch, + nth_process=self.nth_process, + ) + gfo_wrapper_model.pass_through = self.pass_through + + memory_warm_start = self.hg_conv.conv_memory_warm_start( + self.memory_warm_start + ) + + gfo_objective_function = gfo_wrapper_model(self.s_space()) + + self.gfo_optimizer.init_search( + gfo_objective_function, + self.n_iter, + self.max_time, + self.max_score, + self.early_stopping, + self.memory, + memory_warm_start, + False, + ) + for nth_iter in range(self.n_iter): + if p_bar: + p_bar.set_description( + "[" + + str(nth_process) + + "] " + + str(self.objective_function.__name__) + + " (" + + self.optimizer_class.name + + ")", + ) + + self.gfo_optimizer.search_step(nth_iter) + if self.gfo_optimizer.stop.check(): + break + + if p_bar: + p_bar.set_postfix( + best_score=str(gfo_wrapper_model.optimizer.score_best), + best_pos=str(gfo_wrapper_model.optimizer.pos_best), + best_iter=str( + gfo_wrapper_model.optimizer.p_bar._best_since_iter + ), + ) + + p_bar.update(1) + p_bar.refresh() + + self.gfo_optimizer.finish_search() + + self.convert_results2hyper() + + self._add_result_attributes( + self.best_para, + self.best_score, + self.gfo_optimizer.p_bar._best_since_iter, + self.eval_times, + self.iter_times, + self.search_data, + self.gfo_optimizer.random_seed, + ) diff --git a/src_old/hyperactive/optimizers/objective_function.py b/src_old/hyperactive/optimizers/objective_function.py new file mode 100644 index 00000000..7e8c80f2 --- /dev/null +++ b/src_old/hyperactive/optimizers/objective_function.py @@ -0,0 +1,51 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .dictionary import DictClass + + +def gfo2hyper(search_space, para): + values_dict = {} + for _, key in enumerate(search_space.keys()): + pos_ = int(para[key]) + values_dict[key] = search_space[key][pos_] + + return values_dict + + +class ObjectiveFunction(DictClass): + def __init__(self, objective_function, optimizer, callbacks, catch, nth_process): + super().__init__() + + self.objective_function = objective_function + self.optimizer = optimizer + self.callbacks = callbacks + self.catch = catch + self.nth_process = nth_process + + self.nth_iter = 0 + + def run_callbacks(self, type_): + if self.callbacks and type_ in self.callbacks: + [callback(self) for callback in self.callbacks[type_]] + + def __call__(self, search_space): + # wrapper for GFOs + def _model(para): + self.nth_iter = len(self.optimizer.pos_l) + para = gfo2hyper(search_space, para) + self.para_dict = para + + try: + self.run_callbacks("before") + results = self.objective_function(self) + self.run_callbacks("after") + except tuple(self.catch.keys()) as e: + results = self.catch[e.__class__] + + return results + + _model.__name__ = self.objective_function.__name__ + return _model diff --git a/src_old/hyperactive/optimizers/optimizer_attributes.py b/src_old/hyperactive/optimizers/optimizer_attributes.py new file mode 100644 index 00000000..4ef67b7b --- /dev/null +++ b/src_old/hyperactive/optimizers/optimizer_attributes.py @@ -0,0 +1,32 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class OptimizerAttributes: + def __init__(self): + self.best_para = None + self.best_score = None + self.best_since_iter = None + self.eval_times = None + self.iter_times = None + self.search_data = None + self.random_seed = None + + def _add_result_attributes( + self, + best_para, + best_score, + best_since_iter, + eval_times, + iter_times, + search_data, + random_seed, + ): + self.best_para = best_para + self.best_score = best_score + self.best_since_iter = best_since_iter + self.eval_times = eval_times + self.iter_times = iter_times + self.search_data = search_data + self.random_seed = random_seed diff --git a/src_old/hyperactive/optimizers/optimizers.py b/src_old/hyperactive/optimizers/optimizers.py new file mode 100644 index 00000000..61a30a51 --- /dev/null +++ b/src_old/hyperactive/optimizers/optimizers.py @@ -0,0 +1,170 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .hyper_optimizer import HyperOptimizer + +from gradient_free_optimizers import ( + HillClimbingOptimizer as _HillClimbingOptimizer, + StochasticHillClimbingOptimizer as _StochasticHillClimbingOptimizer, + RepulsingHillClimbingOptimizer as _RepulsingHillClimbingOptimizer, + SimulatedAnnealingOptimizer as _SimulatedAnnealingOptimizer, + DownhillSimplexOptimizer as _DownhillSimplexOptimizer, + RandomSearchOptimizer as _RandomSearchOptimizer, + GridSearchOptimizer as _GridSearchOptimizer, + RandomRestartHillClimbingOptimizer as _RandomRestartHillClimbingOptimizer, + RandomAnnealingOptimizer as _RandomAnnealingOptimizer, + PowellsMethod as _PowellsMethod, + PatternSearch as _PatternSearch, + ParallelTemperingOptimizer as _ParallelTemperingOptimizer, + ParticleSwarmOptimizer as _ParticleSwarmOptimizer, + SpiralOptimization as _SpiralOptimization_, + GeneticAlgorithmOptimizer as _GeneticAlgorithmOptimizer, + EvolutionStrategyOptimizer as _EvolutionStrategyOptimizer, + DifferentialEvolutionOptimizer as _DifferentialEvolutionOptimizer, + BayesianOptimizer as _BayesianOptimizer, + LipschitzOptimizer as _LipschitzOptimizer_, + DirectAlgorithm as _DirectAlgorithm_, + TreeStructuredParzenEstimators as _TreeStructuredParzenEstimators, + ForestOptimizer as _ForestOptimizer, + EnsembleOptimizer as _EnsembleOptimizer, +) + + +class HillClimbingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _HillClimbingOptimizer + + +class StochasticHillClimbingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _StochasticHillClimbingOptimizer + + +class RepulsingHillClimbingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _RepulsingHillClimbingOptimizer + + +class SimulatedAnnealingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _SimulatedAnnealingOptimizer + + +class DownhillSimplexOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _DownhillSimplexOptimizer + + +class RandomSearchOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _RandomSearchOptimizer + + +class GridSearchOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _GridSearchOptimizer + + +class RandomRestartHillClimbingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _RandomRestartHillClimbingOptimizer + + +class RandomAnnealingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _RandomAnnealingOptimizer + + +class PowellsMethod(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _PowellsMethod + + +class PatternSearch(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _PatternSearch + + +class ParallelTemperingOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _ParallelTemperingOptimizer + + +class ParticleSwarmOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _ParticleSwarmOptimizer + + +class SpiralOptimization(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _SpiralOptimization_ + + +class GeneticAlgorithmOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _GeneticAlgorithmOptimizer + + +class EvolutionStrategyOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _EvolutionStrategyOptimizer + + +class DifferentialEvolutionOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _DifferentialEvolutionOptimizer + + +class BayesianOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _BayesianOptimizer + + +class LipschitzOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _LipschitzOptimizer_ + + +class DirectAlgorithm(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _DirectAlgorithm_ + + +class TreeStructuredParzenEstimators(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _TreeStructuredParzenEstimators + + +class ForestOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _ForestOptimizer + + +class EnsembleOptimizer(HyperOptimizer): + def __init__(self, **opt_params): + super().__init__(**opt_params) + self.optimizer_class = _EnsembleOptimizer diff --git a/src_old/hyperactive/optimizers/strategies/__init__.py b/src_old/hyperactive/optimizers/strategies/__init__.py new file mode 100644 index 00000000..805bb629 --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/__init__.py @@ -0,0 +1,11 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .custom_optimization_strategy import CustomOptimizationStrategy + + +__all__ = [ + "CustomOptimizationStrategy", +] diff --git a/src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py b/src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py new file mode 100644 index 00000000..ccf7c5f1 --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py @@ -0,0 +1,22 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from .optimization_strategy import BaseOptimizationStrategy + + +class CustomOptimizationStrategy(BaseOptimizationStrategy): + def __init__(self): + super().__init__() + + self.optimizer_setup_l = [] + self.duration_sum = 0 + + def add_optimizer(self, optimizer, duration=1, early_stopping=None): + self.duration_sum += duration + optimizer_setup = { + "optimizer": optimizer, + "duration": duration, + "early_stopping": early_stopping, + } + self.optimizer_setup_l.append(optimizer_setup) diff --git a/src_old/hyperactive/optimizers/strategies/optimization_strategy.py b/src_old/hyperactive/optimizers/strategies/optimization_strategy.py new file mode 100644 index 00000000..0be4d00e --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/optimization_strategy.py @@ -0,0 +1,127 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .optimizer_attributes import OptimizerAttributes + + +class BaseOptimizationStrategy(OptimizerAttributes): + def __init__(self): + super().__init__() + + def setup_search( + self, + objective_function, + s_space, + n_iter, + initialize, + constraints, + pass_through, + callbacks, + catch, + max_score, + early_stopping, + random_state, + memory, + memory_warm_start, + verbosity, + ): + self.objective_function = objective_function + self.s_space = s_space + self.n_iter = n_iter + + self.initialize = initialize + self.constraints = constraints + self.pass_through = pass_through + self.callbacks = callbacks + self.catch = catch + self.max_score = max_score + self.early_stopping = early_stopping + self.random_state = random_state + self.memory = memory + self.memory_warm_start = memory_warm_start + self.verbosity = verbosity + + self._max_time = None + + if "progress_bar" in self.verbosity: + self.verbosity = [] + else: + self.verbosity = [] + + @property + def max_time(self): + return self._max_time + + @max_time.setter + def max_time(self, value): + self._max_time = value + + for optimizer_setup in self.optimizer_setup_l: + optimizer_setup["optimizer"].max_time = value + + def search(self, nth_process, p_bar): + for optimizer_setup in self.optimizer_setup_l: + hyper_opt = optimizer_setup["optimizer"] + duration = optimizer_setup["duration"] + opt_strat_early_stopping = optimizer_setup["early_stopping"] + + if opt_strat_early_stopping: + early_stopping = opt_strat_early_stopping + else: + early_stopping = self.early_stopping + + n_iter = round(self.n_iter * duration / self.duration_sum) + + # initialize + if self.best_para is not None: + initialize = {} + if "warm_start" in initialize: + initialize["warm_start"].append(self.best_para) + else: + initialize["warm_start"] = [self.best_para] + else: + initialize = dict(self.initialize) + + # memory_warm_start + if self.search_data is not None: + memory_warm_start = self.search_data + else: + memory_warm_start = self.memory_warm_start + + # warm_start_smbo + if ( + hyper_opt.optimizer_class.optimizer_type == "sequential" + and self.search_data is not None + ): + hyper_opt.opt_params["warm_start_smbo"] = self.search_data + + hyper_opt.setup_search( + objective_function=self.objective_function, + s_space=self.s_space, + n_iter=n_iter, + initialize=initialize, + constraints=self.constraints, + pass_through=self.pass_through, + callbacks=self.callbacks, + catch=self.catch, + max_score=self.max_score, + early_stopping=early_stopping, + random_state=self.random_state, + memory=self.memory, + memory_warm_start=memory_warm_start, + verbosity=self.verbosity, + ) + + hyper_opt.search(nth_process, p_bar) + + self._add_result_attributes( + hyper_opt.best_para, + hyper_opt.best_score, + hyper_opt.best_since_iter, + hyper_opt.eval_times, + hyper_opt.iter_times, + hyper_opt.search_data, + hyper_opt.gfo_optimizer.random_seed, + ) diff --git a/src_old/hyperactive/optimizers/strategies/optimizer_attributes.py b/src_old/hyperactive/optimizers/strategies/optimizer_attributes.py new file mode 100644 index 00000000..894f0c13 --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/optimizer_attributes.py @@ -0,0 +1,64 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import pandas as pd + + +class OptimizerAttributes: + def __init__(self): + self.best_para = None + self.best_score = None + self.best_since_iter = None + self.eval_times = None + self.iter_times = None + self.search_data = None + self.random_seed = None + + def _add_result_attributes( + self, + best_para, + best_score, + best_since_iter, + eval_times, + iter_times, + search_data, + random_seed, + ): + if self.best_para is None: + self.best_para = best_para + else: + if best_score > self.best_score: + self.best_para = best_para + + if self.best_score is None: + self.best_score = best_score + else: + if best_score > self.best_score: + self.best_score = best_score + + if self.best_since_iter is None: + self.best_since_iter = best_since_iter + else: + if best_score > self.best_score: + self.best_since_iter = best_since_iter + + if self.eval_times is None: + self.eval_times = eval_times + else: + self.eval_times = self.eval_times + eval_times + + if self.iter_times is None: + self.iter_times = iter_times + else: + self.iter_times = self.iter_times + eval_times + + if self.search_data is None: + self.search_data = search_data + else: + self.search_data = pd.concat( + [self.search_data, search_data], ignore_index=True + ) + + if self.random_seed is None: + self.random_seed = random_seed diff --git a/src_old/hyperactive/print_results.py b/src_old/hyperactive/print_results.py new file mode 100644 index 00000000..e5263e84 --- /dev/null +++ b/src_old/hyperactive/print_results.py @@ -0,0 +1,169 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import logging +import numpy as np + +indent = " " + + +class PrintResults: + def __init__(self, opt_pros, verbosity): + self.opt_pros = opt_pros + self.verbosity = verbosity + + def _print_times(self, eval_time, iter_time, n_iter): + opt_time = iter_time - eval_time + iterPerSec = n_iter / iter_time + + print( + indent, + "Evaluation time :", + eval_time, + "sec", + indent, + "[{} %]".format(round(eval_time / iter_time * 100, 2)), + ) + print( + indent, + "Optimization time :", + opt_time, + "sec", + indent, + "[{} %]".format(round(opt_time / iter_time * 100, 2)), + ) + if iterPerSec >= 1: + print( + indent, + "Iteration time :", + iter_time, + "sec", + indent, + "[{} iter/sec]".format(round(iterPerSec, 2)), + ) + else: + secPerIter = iter_time / n_iter + print( + indent, + "Iteration time :", + iter_time, + "sec", + indent, + "[{} sec/iter]".format(round(secPerIter, 2)), + ) + print(" ") + + def align_para_names(self, para_names): + str_lengths = [len(str_) for str_ in para_names] + max_length = max(str_lengths) + + para_names_align = {} + for para_name, str_length in zip(para_names, str_lengths): + added_spaces = max_length - str_length + para_names_align[para_name] = " " * added_spaces + + return para_names_align + + def _print_results( + self, + objective_function, + best_score, + best_para, + best_iter, + best_additional_results, + random_seed, + ): + print("\nResults: '{}'".format(objective_function.__name__), " ") + if best_para is None: + print(indent, "Best score:", best_score, " ") + print(indent, "Best parameter set:", best_para, " ") + print(indent, "Best iteration:", best_iter, " ") + + else: + print(indent, "Best score:", best_score, " ") + + if best_additional_results: + print(indent, "Best additional results:") + add_results_names = list(best_additional_results.keys()) + add_results_names_align = self.align_para_names(add_results_names) + + for best_additional_result in best_additional_results.keys(): + added_spaces = add_results_names_align[best_additional_result] + print( + indent, + indent, + "'{}'".format(best_additional_result), + "{}:".format(added_spaces), + best_additional_results[best_additional_result], + " ", + ) + + if best_para: + print(indent, "Best parameter set:") + para_names = list(best_para.keys()) + para_names_align = self.align_para_names(para_names) + + for para_key in best_para.keys(): + added_spaces = para_names_align[para_key] + print( + indent, + indent, + "'{}'".format(para_key), + "{}:".format(added_spaces), + best_para[para_key], + " ", + ) + + print(indent, "Best iteration:", best_iter, " ") + + print(" ") + print(indent, "Random seed:", random_seed, " ") + print(" ") + + def print_process(self, results, nth_process): + verbosity = self.verbosity + objective_function = self.opt_pros[nth_process].objective_function + search_space = self.opt_pros[nth_process].s_space.search_space + + search_data = results["search_data"] + + try: + best_sample = search_data.iloc[search_data["score"].idxmax()] + + except TypeError: + logging.warning( + "Warning: Cannot index by location index with a non-integer key" + ) + + else: + best_score = best_sample["score"] + best_values = best_sample[list(search_space.keys())] + best_para = dict(zip(list(search_space.keys()), best_values)) + best_additional_results_df = best_sample.drop( + ["score"] + list(search_space.keys()) + ) + best_additional_results = best_additional_results_df.to_dict() + + best_iter = results["best_iter"] + eval_times = results["eval_times"] + iter_times = results["iter_times"] + random_seed = results["random_seed"] + + n_iter = self.opt_pros[nth_process].n_iter + + eval_time = np.array(eval_times).sum() + iter_time = np.array(iter_times).sum() + + if "print_results" in verbosity: + self._print_results( + objective_function, + best_score, + best_para, + best_iter, + best_additional_results, + random_seed, + ) + + if "print_times" in verbosity: + self._print_times(eval_time, iter_time, n_iter) diff --git a/src_old/hyperactive/process.py b/src_old/hyperactive/process.py new file mode 100644 index 00000000..f008dc80 --- /dev/null +++ b/src_old/hyperactive/process.py @@ -0,0 +1,36 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from tqdm import tqdm + + +def _process_(nth_process, optimizer): + if "progress_bar" in optimizer.verbosity: + p_bar = tqdm( + position=nth_process, + total=optimizer.n_iter, + ascii=" ─", + colour="Yellow", + ) + else: + p_bar = None + + optimizer.search(nth_process, p_bar) + + if p_bar: + p_bar.colour = "GREEN" + p_bar.refresh() + p_bar.close() + + return { + "nth_process": nth_process, + "best_para": optimizer.best_para, + "best_score": optimizer.best_score, + "best_iter": optimizer.best_since_iter, + "eval_times": optimizer.eval_times, + "iter_times": optimizer.iter_times, + "search_data": optimizer.search_data, + "random_seed": optimizer.random_seed, + } diff --git a/src_old/hyperactive/results.py b/src_old/hyperactive/results.py new file mode 100644 index 00000000..9b014e2b --- /dev/null +++ b/src_old/hyperactive/results.py @@ -0,0 +1,88 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +import numpy as np +import pandas as pd + + +class Results: + def __init__(self, results_list, opt_pros): + self.results_list = results_list + self.opt_pros = opt_pros + + self.objFunc2results = {} + self.search_id2results = {} + + def _sort_results_objFunc(self, objective_function): + best_score = -np.inf + best_para = None + search_data = None + + search_data_list = [] + + for results_ in self.results_list: + nth_process = results_["nth_process"] + + opt = self.opt_pros[nth_process] + objective_function_ = opt.objective_function + search_space_ = opt.s_space() + params = list(search_space_.keys()) + + if objective_function_ != objective_function: + continue + + if results_["best_score"] > best_score: + best_score = results_["best_score"] + best_para = results_["best_para"] + + search_data = results_["search_data"] + search_data["eval_times"] = results_["eval_times"] + search_data["iter_times"] = results_["iter_times"] + + search_data_list.append(search_data) + + if len(search_data_list) > 0: + search_data = pd.concat(search_data_list) + + self.objFunc2results[objective_function] = { + "best_para": best_para, + "best_score": best_score, + "search_data": search_data, + "params": params, + } + + def _get_result(self, id_, result_name): + if id_ not in self.objFunc2results: + self._sort_results_objFunc(id_) + + search_data = self.objFunc2results[id_][result_name] + + return search_data + + def best_para(self, id_): + best_para_ = self._get_result(id_, "best_para") + + if best_para_ is not None: + return best_para_ + + raise ValueError("objective function name not recognized") + + def best_score(self, id_): + best_score_ = self._get_result(id_, "best_score") + + if best_score_ != -np.inf: + return best_score_ + + raise ValueError("objective function name not recognized") + + def search_data(self, id_): + search_data = self._get_result(id_, "search_data") + + params = self.objFunc2results[id_]["params"] + + if search_data is not None: + return search_data + + raise ValueError("objective function name not recognized") diff --git a/src_old/hyperactive/run_search.py b/src_old/hyperactive/run_search.py new file mode 100644 index 00000000..130e6207 --- /dev/null +++ b/src_old/hyperactive/run_search.py @@ -0,0 +1,57 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .distribution import ( + single_process, + joblib_wrapper, + multiprocessing_wrapper, + pathos_wrapper, +) +from .process import _process_ + + +def proxy(args): + return _process_(*args) + + +dist_dict = { + "joblib": (joblib_wrapper, _process_), + "multiprocessing": (multiprocessing_wrapper, proxy), + "pathos": (pathos_wrapper, proxy), +} + + +def _get_distribution(distribution): + if hasattr(distribution, "__call__"): + return (distribution, _process_), {} + + elif isinstance(distribution, dict): + dist_key = list(distribution.keys())[0] + dist_paras = list(distribution.values())[0] + + return dist_dict[dist_key], dist_paras + + elif isinstance(distribution, str): + return dist_dict[distribution], {} + + +def run_search(opt_pros, distribution, n_processes): + opts = list(opt_pros.values()) + + processes = list(opt_pros.keys()) + optimizers = list(opt_pros.values()) + process_infos = list(zip(processes, optimizers)) + + if n_processes == "auto": + n_processes = len(process_infos) + + if n_processes == 1: + results_list = single_process(_process_, process_infos) + else: + (distribution, process_func), dist_paras = _get_distribution(distribution) + + results_list = distribution(process_func, process_infos, n_processes) + + return results_list diff --git a/src_old/hyperactive/search_space.py b/src_old/hyperactive/search_space.py new file mode 100644 index 00000000..101201d8 --- /dev/null +++ b/src_old/hyperactive/search_space.py @@ -0,0 +1,134 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import numpy as np + + +class DictClass: + def __init__(self, search_space): + self.search_space = search_space + + def __getitem__(self, key): + return self.search_space[key] + + def keys(self): + return self.search_space.keys() + + def values(self): + return self.search_space.values() + + +class SearchSpace(DictClass): + def __init__(self, search_space): + super().__init__(search_space) + self.search_space = search_space + + self.dim_keys = list(search_space.keys()) + self.values_l = list(self.search_space.values()) + + positions = {} + for key in search_space.keys(): + positions[key] = np.array(range(len(search_space[key]))) + self.positions = positions + + self.check_list() + self.check_non_num_values() + + self.data_types = self.dim_types() + self.func2str = self._create_num_str_ss() + + def __call__(self): + return self.search_space + + def dim_types(self): + data_types = {} + for dim_key in self.dim_keys: + dim_values = np.array(list(self.search_space[dim_key])) + try: + np.subtract(dim_values, dim_values) + np.array(dim_values).searchsorted(dim_values) + except: + _type_ = "object" + else: + _type_ = "number" + + data_types[dim_key] = _type_ + return data_types + + def _create_num_str_ss(self): + func2str = {} + for dim_key in self.dim_keys: + if self.data_types[dim_key] == "number": + func2str[dim_key] = self.search_space[dim_key] + else: + func2str[dim_key] = [] + + dim_values = self.search_space[dim_key] + for value in dim_values: + try: + func_name = value.__name__ + except: + func_name = value + + func2str[dim_key].append(func_name) + return func2str + + def check_list(self): + for dim_key in self.dim_keys: + search_dim = self.search_space[dim_key] + + err_msg = "\n Value in '{}' of search space dictionary must be of type list \n".format( + dim_key + ) + if not isinstance(search_dim, list): + raise ValueError(err_msg) + + @staticmethod + def is_function(value): + try: + value.__name__ + except: + return False + else: + return True + + @staticmethod + def is_number(value): + try: + float(value) + value * 0.1 + value - 0.1 + value / 0.1 + except: + return False + else: + return True + + def _string_or_object(self, dim_key, dim_values): + for dim_value in dim_values: + is_str = isinstance(dim_value, str) + is_func = self.is_function(dim_value) + is_number = self.is_number(dim_value) + + if not is_str and not is_func and not is_number: + msg = "\n The value '{}' of type '{}' in the search space dimension '{}' must be number, string or function \n".format( + dim_value, type(dim_value), dim_key + ) + raise ValueError(msg) + + def check_non_num_values(self): + for dim_key in self.dim_keys: + dim_values = np.array(list(self.search_space[dim_key])) + + try: + np.subtract(dim_values, dim_values) + np.array(dim_values).searchsorted(dim_values) + except: + self._string_or_object(dim_key, dim_values) + else: + if dim_values.ndim != 1: + msg = "Array-like object in '{}' must be one dimensional".format( + dim_key + ) + raise ValueError(msg) From 8f4fdc68262589c7cb02827c3bdb10fd65d5c3e7 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 26 Jan 2025 08:33:34 +0100 Subject: [PATCH 10/62] add src dir for v5 API --- src/hyperactive/__init__.py | 8 - src/hyperactive/base/__init__.py | 5 +- src/hyperactive/base/_experiment.py | 9 +- src/hyperactive/base/_optimizer.py | 33 ---- .../base/_search_space_optional.py | 25 --- .../integrations/sktime/__init__.py | 6 + src/hyperactive/integrations/sktime/main.py | 8 + src/hyperactive/memory/__init__.py | 0 src/hyperactive/memory/memory.py | 0 src/hyperactive/optimizers/_optimizer_api.py | 146 ++++++++++++++++++ .../backend_stuff}/distribution.py | 0 .../backend_stuff}/hyperactive.py | 0 .../backend_stuff}/print_results.py | 0 .../{ => optimizers/backend_stuff}/process.py | 0 .../{ => optimizers/backend_stuff}/results.py | 0 .../backend_stuff}/run_search.py | 0 .../backend_stuff}/search_space.py | 0 .../optimizers/objective_function.py | 7 +- src/hyperactive/optimizers/optimizers.py | 3 +- src/hyperactive/search_config/__init__.py | 3 + src/hyperactive/search_config/_properties.py | 28 ++++ .../search_config/_search_config.py | 108 +++++++++++++ 22 files changed, 314 insertions(+), 75 deletions(-) delete mode 100644 src/hyperactive/base/_search_space_optional.py create mode 100644 src/hyperactive/integrations/sktime/__init__.py create mode 100644 src/hyperactive/integrations/sktime/main.py create mode 100644 src/hyperactive/memory/__init__.py create mode 100644 src/hyperactive/memory/memory.py create mode 100644 src/hyperactive/optimizers/_optimizer_api.py rename src/hyperactive/{ => optimizers/backend_stuff}/distribution.py (100%) rename src/hyperactive/{ => optimizers/backend_stuff}/hyperactive.py (100%) rename src/hyperactive/{ => optimizers/backend_stuff}/print_results.py (100%) rename src/hyperactive/{ => optimizers/backend_stuff}/process.py (100%) rename src/hyperactive/{ => optimizers/backend_stuff}/results.py (100%) rename src/hyperactive/{ => optimizers/backend_stuff}/run_search.py (100%) rename src/hyperactive/{ => optimizers/backend_stuff}/search_space.py (100%) create mode 100644 src/hyperactive/search_config/__init__.py create mode 100644 src/hyperactive/search_config/_properties.py create mode 100644 src/hyperactive/search_config/_search_config.py diff --git a/src/hyperactive/__init__.py b/src/hyperactive/__init__.py index bc82b676..18bbdb16 100644 --- a/src/hyperactive/__init__.py +++ b/src/hyperactive/__init__.py @@ -6,11 +6,3 @@ __version__ = importlib.metadata.version("hyperactive") __license__ = "MIT" - - -from .hyperactive import Hyperactive - - -__all__ = [ - "Hyperactive", -] diff --git a/src/hyperactive/base/__init__.py b/src/hyperactive/base/__init__.py index bf0307d4..a237c3fe 100644 --- a/src/hyperactive/base/__init__.py +++ b/src/hyperactive/base/__init__.py @@ -1,6 +1,5 @@ """Base classes for optimizers and experiments.""" -from hyperactive.base._experiment import BaseExperiment -from hyperactive.base._optimizer import BaseOptimizer +from ._experiment import BaseExperiment -__all__ = ["BaseExperiment", "BaseOptimizer"] +__all__ = ["BaseExperiment"] diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index ad3a5999..8b163e65 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -6,9 +6,16 @@ class BaseExperiment(BaseObject): """Base class for experiment.""" - def __init__(self): + def __init__( + self, + callbacks: Dict[str, callable] = None, + catch: Dict = None, + ): super().__init__() + self.callbacks = callbacks + self.catch = catch + def __call__(self, **kwargs): """Score parameters, with kwargs call.""" return self.score(kwargs) diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py index 55bf7d1a..e69de29b 100644 --- a/src/hyperactive/base/_optimizer.py +++ b/src/hyperactive/base/_optimizer.py @@ -1,33 +0,0 @@ -"""Base class for optimizer.""" - -from skbase.base import BaseObject - - -class BaseOptimizer(BaseObject): - """Base class for optimizer.""" - - def __init__(self): - super().__init__() - - def add_search(self, experiment, search_config: dict): - """Add a new optimization search process with specified parameters. - - Parameters - ---------- - experiment : BaseExperiment - The experiment to optimize parameters for. - search_config : dict with str keys - The search configuration dictionary. - """ - self.experiment = experiment - self.search_config = search_config - - def run(self, max_time=None): - """Run the optimization search process. - - Parameters - ---------- - max_time : float - The maximum time used for the optimization process. - """ - raise NotImplementedError diff --git a/src/hyperactive/base/_search_space_optional.py b/src/hyperactive/base/_search_space_optional.py deleted file mode 100644 index 2a17dd3d..00000000 --- a/src/hyperactive/base/_search_space_optional.py +++ /dev/null @@ -1,25 +0,0 @@ -import numpy as np -from hyperactive import BaseSearchSpace - - -""" -Optional: - -It might make sense to use a class instead of a dictionary for the search-space. -The search-space can have specifc properties, that can be computed from the params (previously called search-space-dictionary). -The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. -""" - - -class SearchSpace: - search_space: dict = None - - def __init__(self, **params): - super().__init__() - - for key, value in params.items(): - setattr(self, key, value) - self.search_space[key] = value - - def __call__(self): - return self.search_space diff --git a/src/hyperactive/integrations/sktime/__init__.py b/src/hyperactive/integrations/sktime/__init__.py new file mode 100644 index 00000000..09bdbd71 --- /dev/null +++ b/src/hyperactive/integrations/sktime/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .main import HyperactiveSearchCV diff --git a/src/hyperactive/integrations/sktime/main.py b/src/hyperactive/integrations/sktime/main.py new file mode 100644 index 00000000..f0d7c5db --- /dev/null +++ b/src/hyperactive/integrations/sktime/main.py @@ -0,0 +1,8 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class HyperactiveSearchCV: + def __init__(self) -> None: + pass diff --git a/src/hyperactive/memory/__init__.py b/src/hyperactive/memory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hyperactive/memory/memory.py b/src/hyperactive/memory/memory.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py new file mode 100644 index 00000000..e3d2418e --- /dev/null +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -0,0 +1,146 @@ +"""Base class for optimizer.""" + +import numpy as np +from typing import Union, List, Dict, Type +import copy +import multiprocessing as mp +import pandas as pd + +from .objective_function import ObjectiveFunction +from .hyper_gradient_conv import HyperGradientConv +from .optimizer_attributes import OptimizerAttributes +from .constraint import Constraint +from .backend_stuff.search_space import SearchSpace +from .optimizer_attributes import OptimizerAttributes + + +from skbase.base import BaseObject + + +class BaseOptimizer(BaseObject, OptimizerAttributes): + """Base class for optimizer.""" + + def __init__(self, **opt_params): + super().__init__() + self.opt_params = opt_params + + def convert_results2hyper(self): + self.eval_times = sum(self.gfo_optimizer.eval_times) + self.iter_times = sum(self.gfo_optimizer.iter_times) + + if self.gfo_optimizer.best_para is not None: + value = self.hg_conv.para2value(self.gfo_optimizer.best_para) + position = self.hg_conv.position2value(value) + best_para = self.hg_conv.value2para(position) + self.best_para = best_para + else: + self.best_para = None + + self.best_score = self.gfo_optimizer.best_score + self.positions = self.gfo_optimizer.search_data + self.search_data = self.hg_conv.positions2results(self.positions) + + results_dd = self.gfo_optimizer.search_data.drop_duplicates( + subset=self.s_space.dim_keys, keep="first" + ) + self.memory_values_df = results_dd[ + self.s_space.dim_keys + ["score"] + ].reset_index(drop=True) + + def _setup_process(self): + self.hg_conv = HyperGradientConv(self.s_space) + + search_space_positions = self.s_space.positions + + # conv warm start for smbo from values into positions + if "warm_start_smbo" in self.opt_params: + self.opt_params["warm_start_smbo"] = ( + self.hg_conv.conv_memory_warm_start( + self.opt_params["warm_start_smbo"] + ) + ) + + self.gfo_optimizer = self.optimizer_class( + search_space=search_space_positions, + **self.opt_params, + ) + + self.conv = self.gfo_optimizer.conv + + def add_search( + self, + experiment, + search_config: dict, + n_iter: int, + search_id=None, + n_jobs: int = 1, + initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, + ): + """Add a new optimization search process with specified parameters. + + Parameters + ---------- + experiment : BaseExperiment + The experiment to optimize parameters for. + search_config : dict with str keys + The search configuration dictionary. + """ + self.experiment = experiment + self.search_config = search_config + self.n_iter = n_iter + self.search_id = search_id + self.n_jobs = n_jobs + self.initialize = initialize + + search_id = self._default_search_id(search_id, experiment._score) + s_space = SearchSpace(search_config.search_space) + + n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs + + for _ in range(n_jobs): + nth_process = len(self.opt_pros) + self.opt_pros[nth_process] = optimizer + + def run( + self, + max_time=None, + max_score=None, + early_stopping=None, + ): + self._setup_process() + + gfo_wrapper_model = ObjectiveFunction( + objective_function=self.experiment._score, + callbacks=[], + catch={}, + ) + + gfo_objective_function = gfo_wrapper_model(self.s_space()) + + self.gfo_optimizer.init_search( + gfo_objective_function, + self.n_iter, + max_time, + max_score, + early_stopping, + False, + ) + for nth_iter in range(self.n_iter): + print("iterate") + self.gfo_optimizer.search_step(nth_iter) + if self.gfo_optimizer.stop.check(): + break + + self.gfo_optimizer.finish_search() + + self.convert_results2hyper() + + self._add_result_attributes( + self.best_para, + self.best_score, + self.gfo_optimizer.p_bar._best_since_iter, + self.eval_times, + self.iter_times, + self.search_data, + self.gfo_optimizer.random_seed, + ) diff --git a/src/hyperactive/distribution.py b/src/hyperactive/optimizers/backend_stuff/distribution.py similarity index 100% rename from src/hyperactive/distribution.py rename to src/hyperactive/optimizers/backend_stuff/distribution.py diff --git a/src/hyperactive/hyperactive.py b/src/hyperactive/optimizers/backend_stuff/hyperactive.py similarity index 100% rename from src/hyperactive/hyperactive.py rename to src/hyperactive/optimizers/backend_stuff/hyperactive.py diff --git a/src/hyperactive/print_results.py b/src/hyperactive/optimizers/backend_stuff/print_results.py similarity index 100% rename from src/hyperactive/print_results.py rename to src/hyperactive/optimizers/backend_stuff/print_results.py diff --git a/src/hyperactive/process.py b/src/hyperactive/optimizers/backend_stuff/process.py similarity index 100% rename from src/hyperactive/process.py rename to src/hyperactive/optimizers/backend_stuff/process.py diff --git a/src/hyperactive/results.py b/src/hyperactive/optimizers/backend_stuff/results.py similarity index 100% rename from src/hyperactive/results.py rename to src/hyperactive/optimizers/backend_stuff/results.py diff --git a/src/hyperactive/run_search.py b/src/hyperactive/optimizers/backend_stuff/run_search.py similarity index 100% rename from src/hyperactive/run_search.py rename to src/hyperactive/optimizers/backend_stuff/run_search.py diff --git a/src/hyperactive/search_space.py b/src/hyperactive/optimizers/backend_stuff/search_space.py similarity index 100% rename from src/hyperactive/search_space.py rename to src/hyperactive/optimizers/backend_stuff/search_space.py diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/objective_function.py index 7e8c80f2..dc7abcca 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/objective_function.py @@ -16,14 +16,12 @@ def gfo2hyper(search_space, para): class ObjectiveFunction(DictClass): - def __init__(self, objective_function, optimizer, callbacks, catch, nth_process): + def __init__(self, objective_function, callbacks, catch): super().__init__() self.objective_function = objective_function - self.optimizer = optimizer self.callbacks = callbacks self.catch = catch - self.nth_process = nth_process self.nth_iter = 0 @@ -34,11 +32,12 @@ def run_callbacks(self, type_): def __call__(self, search_space): # wrapper for GFOs def _model(para): - self.nth_iter = len(self.optimizer.pos_l) + # self.nth_iter = len(self.optimizer.pos_l) para = gfo2hyper(search_space, para) self.para_dict = para try: + print("try eval") self.run_callbacks("before") results = self.objective_function(self) self.run_callbacks("after") diff --git a/src/hyperactive/optimizers/optimizers.py b/src/hyperactive/optimizers/optimizers.py index 61a30a51..2b5a1e80 100644 --- a/src/hyperactive/optimizers/optimizers.py +++ b/src/hyperactive/optimizers/optimizers.py @@ -4,6 +4,7 @@ from .hyper_optimizer import HyperOptimizer +from ._optimizer_api import BaseOptimizer from gradient_free_optimizers import ( HillClimbingOptimizer as _HillClimbingOptimizer, @@ -32,7 +33,7 @@ ) -class HillClimbingOptimizer(HyperOptimizer): +class HillClimbingOptimizer(BaseOptimizer): def __init__(self, **opt_params): super().__init__(**opt_params) self.optimizer_class = _HillClimbingOptimizer diff --git a/src/hyperactive/search_config/__init__.py b/src/hyperactive/search_config/__init__.py new file mode 100644 index 00000000..346e9a27 --- /dev/null +++ b/src/hyperactive/search_config/__init__.py @@ -0,0 +1,3 @@ +from ._search_config import SearchConfig + +__all__ = ["SearchConfig"] diff --git a/src/hyperactive/search_config/_properties.py b/src/hyperactive/search_config/_properties.py new file mode 100644 index 00000000..aa1cf543 --- /dev/null +++ b/src/hyperactive/search_config/_properties.py @@ -0,0 +1,28 @@ +import numpy as np + + +def n_dim(search_space): + return len(search_space) + + +def dim_names(search_space): + return list(search_space.keys()) + + +def position_space(search_space): + return { + key: np.array(range(len(search_space[key]))) + for key in search_space.keys() + } + + +def calculate_properties(func): + def wrapper(self, *args, **kwargs): + func(self, *args, **kwargs) + + self.n_dim = n_dim(self._search_space) + self.dim_names = dim_names(self._search_space) + self.position_space = position_space(self._search_space) + print(" ---> search-space updated!") + + return wrapper diff --git a/src/hyperactive/search_config/_search_config.py b/src/hyperactive/search_config/_search_config.py new file mode 100644 index 00000000..e2e3fda0 --- /dev/null +++ b/src/hyperactive/search_config/_search_config.py @@ -0,0 +1,108 @@ +""" +Optional: + +It might make sense to use a class instead of a dictionary for the search-space. +The search-space can have specifc properties, that can be computed from the params (previously called search-space-dictionary). +The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. +""" + +from collections.abc import MutableMapping + +from ._properties import calculate_properties + + +class SearchSpaceDictLike(MutableMapping): + _search_space: dict = None + + def __init__(self, constraints: List[callable] = None, **param_space): + self._search_space = dict(**param_space) + self.constraints = constraints + + def __getitem__(self, key): + return self._search_space[key] + + def __setitem__(self, key, value): + self._search_space[key] = value + + def __delitem__(self, key): + del self._search_space[key] + + def __iter__(self): + return iter(self._search_space) + + def __len__(self): + return len(self._search_space) + + def __repr__(self): + return f"{self.__class__.__name__}({self._search_space})" + + def __call__(self): + return self._search_space + + def keys(self): + return self._search_space.keys() + + def values(self): + return self._search_space.values() + + def items(self): + return self._search_space.items() + + +class SearchConfig(SearchSpaceDictLike): + + @calculate_properties + def __init__(self, **param_space): + super().__init__(**param_space) + + for key, value in param_space.items(): + setattr(self, key, value) + + @calculate_properties + def __setitem__(self, key, value): + SearchSpaceDictLike.__setitem__(self, key, value) + + @calculate_properties + def __delitem__(self, key): + SearchSpaceDictLike.__delitem__(self, key) + + def print(self, indent=2, max_list_length=5): + """ + Prints the dictionary in a readable format. + + Args: + indent (int): The number of spaces to indent nested structures. + max_list_length (int): The maximum number of items to display from long lists. + """ + + def format_value(value, level=0): + prefix = " " * (level * indent) + if isinstance(value, list): + if len(value) > max_list_length: + # Truncate long lists for readability + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value[:max_list_length] + ) + result += f"{prefix}{' ' * indent}... ({len(value) - max_list_length} more items)\n" + result += f"{prefix}]" + else: + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value + ) + result += f"{prefix}]" + elif isinstance(value, dict): + # Format nested dictionaries + result = "{\n" + for k, v in value.items(): + result += f"{prefix}{' ' * indent}{repr(k)}: {format_value(v, level + 1)},\n" + result += f"{prefix}}}" + else: + result = repr(value) + return result + + for key, value in self.items(): + print(f"{key}: {format_value(value)}") From 5580677eb0a8e17bf3bb9554866249eca6abe48a Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 26 Jan 2025 09:02:13 +0100 Subject: [PATCH 11/62] create working prototype of v5 API --- src/hyperactive/base/_experiment.py | 1 + src/hyperactive/optimizers/_optimizer_api.py | 205 +++++++++--------- src/hyperactive/optimizers/hyper_optimizer.py | 13 +- .../optimizers/objective_function.py | 1 - src/hyperactive/optimizers/optimizers.py | 114 ++++------ .../search_config/_search_config.py | 1 + 6 files changed, 155 insertions(+), 180 deletions(-) diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index 8b163e65..f0adea77 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -1,5 +1,6 @@ """Base class for experiment.""" +from typing import Union, List, Dict, Type from skbase.base import BaseObject diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index e3d2418e..8d41183c 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -6,141 +6,142 @@ import multiprocessing as mp import pandas as pd -from .objective_function import ObjectiveFunction -from .hyper_gradient_conv import HyperGradientConv -from .optimizer_attributes import OptimizerAttributes -from .constraint import Constraint from .backend_stuff.search_space import SearchSpace -from .optimizer_attributes import OptimizerAttributes - +from .backend_stuff.run_search import run_search +from .hyper_optimizer import HyperOptimizer +from .backend_stuff.results import Results +from .backend_stuff.print_results import PrintResults from skbase.base import BaseObject -class BaseOptimizer(BaseObject, OptimizerAttributes): +class BaseOptimizer(BaseObject): """Base class for optimizer.""" - def __init__(self, **opt_params): + opt_pros = {} + + def __init__(self, optimizer_class, opt_params): super().__init__() self.opt_params = opt_params + self.hyper_optimizer = HyperOptimizer(optimizer_class, opt_params) - def convert_results2hyper(self): - self.eval_times = sum(self.gfo_optimizer.eval_times) - self.iter_times = sum(self.gfo_optimizer.iter_times) - - if self.gfo_optimizer.best_para is not None: - value = self.hg_conv.para2value(self.gfo_optimizer.best_para) - position = self.hg_conv.position2value(value) - best_para = self.hg_conv.value2para(position) - self.best_para = best_para - else: - self.best_para = None + @staticmethod + def _default_search_id(search_id, objective_function): + if not search_id: + search_id = objective_function.__name__ + return search_id - self.best_score = self.gfo_optimizer.best_score - self.positions = self.gfo_optimizer.search_data - self.search_data = self.hg_conv.positions2results(self.positions) - - results_dd = self.gfo_optimizer.search_data.drop_duplicates( - subset=self.s_space.dim_keys, keep="first" - ) - self.memory_values_df = results_dd[ - self.s_space.dim_keys + ["score"] - ].reset_index(drop=True) + @staticmethod + def check_list(search_space): + for key in search_space.keys(): + search_dim = search_space[key] - def _setup_process(self): - self.hg_conv = HyperGradientConv(self.s_space) - - search_space_positions = self.s_space.positions - - # conv warm start for smbo from values into positions - if "warm_start_smbo" in self.opt_params: - self.opt_params["warm_start_smbo"] = ( - self.hg_conv.conv_memory_warm_start( - self.opt_params["warm_start_smbo"] - ) + error_msg = "Value in '{}' of search space dictionary must be of type list".format( + key ) - - self.gfo_optimizer = self.optimizer_class( - search_space=search_space_positions, - **self.opt_params, - ) - - self.conv = self.gfo_optimizer.conv + if not isinstance(search_dim, list): + print("Warning", error_msg) + # raise ValueError(error_msg) def add_search( self, - experiment, - search_config: dict, + experiment: callable, + search_space: Dict[str, list], n_iter: int, search_id=None, n_jobs: int = 1, + verbosity: list = ["progress_bar", "print_results", "print_times"], initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, + constraints: List[callable] = None, + pass_through: Dict = None, + callbacks: Dict[str, callable] = None, + catch: Dict = None, + max_score: float = None, + early_stopping: Dict = None, + random_state: int = None, + memory: Union[str, bool] = "share", + memory_warm_start: pd.DataFrame = None, ): - """Add a new optimization search process with specified parameters. - - Parameters - ---------- - experiment : BaseExperiment - The experiment to optimize parameters for. - search_config : dict with str keys - The search configuration dictionary. """ - self.experiment = experiment - self.search_config = search_config - self.n_iter = n_iter - self.search_id = search_id - self.n_jobs = n_jobs - self.initialize = initialize + Add a new optimization search process with specified parameters. + + Parameters: + - objective_function: The objective function to optimize. + - search_space: Dictionary defining the search space for optimization. + - n_iter: Number of iterations for the optimization process. + - search_id: Identifier for the search process (default: None). + - n_jobs: Number of parallel jobs to run (default: 1). + - initialize: Dictionary specifying initialization parameters (default: {"grid": 4, "random": 2, "vertices": 4}). + - constraints: List of constraint functions (default: None). + - pass_through: Dictionary of additional parameters to pass through (default: None). + - callbacks: Dictionary of callback functions (default: None). + - catch: Dictionary of exceptions to catch during optimization (default: None). + - max_score: Maximum score to achieve (default: None). + - early_stopping: Dictionary specifying early stopping criteria (default: None). + - random_state: Seed for random number generation (default: None). + - memory: Option to share memory between processes (default: "share"). + - memory_warm_start: DataFrame containing warm start memory (default: None). + """ - search_id = self._default_search_id(search_id, experiment._score) - s_space = SearchSpace(search_config.search_space) + objective_function = experiment._score + + self.check_list(search_space) + + constraints = constraints or [] + pass_through = pass_through or {} + callbacks = callbacks or {} + catch = catch or {} + early_stopping = early_stopping or {} + + search_id = self._default_search_id(search_id, objective_function) + s_space = SearchSpace(search_space) + self.verbosity = verbosity + + self.hyper_optimizer.setup_search( + objective_function=objective_function, + s_space=s_space, + n_iter=n_iter, + initialize=initialize, + constraints=constraints, + pass_through=pass_through, + callbacks=callbacks, + catch=catch, + max_score=max_score, + early_stopping=early_stopping, + random_state=random_state, + memory=memory, + memory_warm_start=memory_warm_start, + verbosity=verbosity, + ) n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs for _ in range(n_jobs): nth_process = len(self.opt_pros) - self.opt_pros[nth_process] = optimizer + self.opt_pros[nth_process] = self.hyper_optimizer + + def _print_info(self): + print_res = PrintResults(self.opt_pros, self.verbosity) + + if self.verbosity: + for _ in range(len(self.opt_pros)): + print("") + + for results in self.results_list: + nth_process = results["nth_process"] + print_res.print_process(results, nth_process) def run( self, max_time=None, - max_score=None, - early_stopping=None, + distribution: str = "multiprocessing", + n_processes: Union[str, int] = "auto", ): - self._setup_process() + for opt in self.opt_pros.values(): + opt.max_time = max_time - gfo_wrapper_model = ObjectiveFunction( - objective_function=self.experiment._score, - callbacks=[], - catch={}, - ) + self.results_list = run_search(self.opt_pros, distribution, n_processes) - gfo_objective_function = gfo_wrapper_model(self.s_space()) + self.results_ = Results(self.results_list, self.opt_pros) - self.gfo_optimizer.init_search( - gfo_objective_function, - self.n_iter, - max_time, - max_score, - early_stopping, - False, - ) - for nth_iter in range(self.n_iter): - print("iterate") - self.gfo_optimizer.search_step(nth_iter) - if self.gfo_optimizer.stop.check(): - break - - self.gfo_optimizer.finish_search() - - self.convert_results2hyper() - - self._add_result_attributes( - self.best_para, - self.best_score, - self.gfo_optimizer.p_bar._best_since_iter, - self.eval_times, - self.iter_times, - self.search_data, - self.gfo_optimizer.random_seed, - ) + self._print_info() diff --git a/src/hyperactive/optimizers/hyper_optimizer.py b/src/hyperactive/optimizers/hyper_optimizer.py index c7710023..a50d4cba 100644 --- a/src/hyperactive/optimizers/hyper_optimizer.py +++ b/src/hyperactive/optimizers/hyper_optimizer.py @@ -11,8 +11,9 @@ class HyperOptimizer(OptimizerAttributes): - def __init__(self, **opt_params): + def __init__(self, optimizer_class, opt_params): super().__init__() + self.optimizer_class = optimizer_class self.opt_params = opt_params def setup_search( @@ -113,10 +114,8 @@ def search(self, nth_process, p_bar): gfo_wrapper_model = ObjectiveFunction( objective_function=self.objective_function, - optimizer=self.gfo_optimizer, callbacks=self.callbacks, catch=self.catch, - nth_process=self.nth_process, ) gfo_wrapper_model.pass_through = self.pass_through @@ -154,11 +153,9 @@ def search(self, nth_process, p_bar): if p_bar: p_bar.set_postfix( - best_score=str(gfo_wrapper_model.optimizer.score_best), - best_pos=str(gfo_wrapper_model.optimizer.pos_best), - best_iter=str( - gfo_wrapper_model.optimizer.p_bar._best_since_iter - ), + best_score=str(self.gfo_optimizer.score_best), + best_pos=str(self.gfo_optimizer.pos_best), + best_iter=str(self.gfo_optimizer.p_bar._best_since_iter), ) p_bar.update(1) diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/objective_function.py index dc7abcca..6995e640 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/objective_function.py @@ -37,7 +37,6 @@ def _model(para): self.para_dict = para try: - print("try eval") self.run_callbacks("before") results = self.objective_function(self) self.run_callbacks("after") diff --git a/src/hyperactive/optimizers/optimizers.py b/src/hyperactive/optimizers/optimizers.py index 2b5a1e80..20874c89 100644 --- a/src/hyperactive/optimizers/optimizers.py +++ b/src/hyperactive/optimizers/optimizers.py @@ -3,7 +3,6 @@ # License: MIT License -from .hyper_optimizer import HyperOptimizer from ._optimizer_api import BaseOptimizer from gradient_free_optimizers import ( @@ -35,137 +34,114 @@ class HillClimbingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _HillClimbingOptimizer + super().__init__(_HillClimbingOptimizer, opt_params) -class StochasticHillClimbingOptimizer(HyperOptimizer): +class StochasticHillClimbingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _StochasticHillClimbingOptimizer + super().__init__(_StochasticHillClimbingOptimizer, opt_params) -class RepulsingHillClimbingOptimizer(HyperOptimizer): +class RepulsingHillClimbingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _RepulsingHillClimbingOptimizer + super().__init__(_RepulsingHillClimbingOptimizer, opt_params) -class SimulatedAnnealingOptimizer(HyperOptimizer): +class SimulatedAnnealingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _SimulatedAnnealingOptimizer + super().__init__(_SimulatedAnnealingOptimizer, opt_params) -class DownhillSimplexOptimizer(HyperOptimizer): +class DownhillSimplexOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _DownhillSimplexOptimizer + super().__init__(_DownhillSimplexOptimizer, opt_params) -class RandomSearchOptimizer(HyperOptimizer): +class RandomSearchOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _RandomSearchOptimizer + super().__init__(_RandomSearchOptimizer, opt_params) -class GridSearchOptimizer(HyperOptimizer): +class GridSearchOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _GridSearchOptimizer + super().__init__(_GridSearchOptimizer, opt_params) -class RandomRestartHillClimbingOptimizer(HyperOptimizer): +class RandomRestartHillClimbingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _RandomRestartHillClimbingOptimizer + super().__init__(_RandomRestartHillClimbingOptimizer, opt_params) -class RandomAnnealingOptimizer(HyperOptimizer): +class RandomAnnealingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _RandomAnnealingOptimizer + super().__init__(_RandomAnnealingOptimizer, opt_params) -class PowellsMethod(HyperOptimizer): +class PowellsMethod(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _PowellsMethod + super().__init__(_PowellsMethod, opt_params) -class PatternSearch(HyperOptimizer): +class PatternSearch(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _PatternSearch + super().__init__(_PatternSearch, opt_params) -class ParallelTemperingOptimizer(HyperOptimizer): +class ParallelTemperingOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _ParallelTemperingOptimizer + super().__init__(_ParallelTemperingOptimizer, opt_params) -class ParticleSwarmOptimizer(HyperOptimizer): +class ParticleSwarmOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _ParticleSwarmOptimizer + super().__init__(_ParticleSwarmOptimizer, opt_params) -class SpiralOptimization(HyperOptimizer): +class SpiralOptimization(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _SpiralOptimization_ + super().__init__(_SpiralOptimization_, opt_params) -class GeneticAlgorithmOptimizer(HyperOptimizer): +class GeneticAlgorithmOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _GeneticAlgorithmOptimizer + super().__init__(_GeneticAlgorithmOptimizer, opt_params) -class EvolutionStrategyOptimizer(HyperOptimizer): +class EvolutionStrategyOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _EvolutionStrategyOptimizer + super().__init__(_EvolutionStrategyOptimizer, opt_params) -class DifferentialEvolutionOptimizer(HyperOptimizer): +class DifferentialEvolutionOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _DifferentialEvolutionOptimizer + super().__init__(_DifferentialEvolutionOptimizer, opt_params) -class BayesianOptimizer(HyperOptimizer): +class BayesianOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _BayesianOptimizer + super().__init__(_BayesianOptimizer, opt_params) -class LipschitzOptimizer(HyperOptimizer): +class LipschitzOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _LipschitzOptimizer_ + super().__init__(_LipschitzOptimizer_, opt_params) -class DirectAlgorithm(HyperOptimizer): +class DirectAlgorithm(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _DirectAlgorithm_ + super().__init__(_DirectAlgorithm_, opt_params) -class TreeStructuredParzenEstimators(HyperOptimizer): +class TreeStructuredParzenEstimators(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _TreeStructuredParzenEstimators + super().__init__(_TreeStructuredParzenEstimators, opt_params) -class ForestOptimizer(HyperOptimizer): +class ForestOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _ForestOptimizer + super().__init__(_ForestOptimizer, opt_params) -class EnsembleOptimizer(HyperOptimizer): +class EnsembleOptimizer(BaseOptimizer): def __init__(self, **opt_params): - super().__init__(**opt_params) - self.optimizer_class = _EnsembleOptimizer + super().__init__(_EnsembleOptimizer, opt_params) diff --git a/src/hyperactive/search_config/_search_config.py b/src/hyperactive/search_config/_search_config.py index e2e3fda0..127d9df0 100644 --- a/src/hyperactive/search_config/_search_config.py +++ b/src/hyperactive/search_config/_search_config.py @@ -6,6 +6,7 @@ The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. """ +from typing import Union, List, Dict, Type from collections.abc import MutableMapping from ._properties import calculate_properties From 87c90761b2997630048154eee64c3cb1d973ad86 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Mon, 27 Jan 2025 19:49:57 +0100 Subject: [PATCH 12/62] add example for new API --- examples/v5_API_example/example.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 examples/v5_API_example/example.py diff --git a/examples/v5_API_example/example.py b/examples/v5_API_example/example.py new file mode 100644 index 00000000..ebe98b6d --- /dev/null +++ b/examples/v5_API_example/example.py @@ -0,0 +1,39 @@ +import numpy as np +from sklearn.datasets import load_diabetes +from sklearn.tree import DecisionTreeRegressor +from sklearn.model_selection import cross_val_score + +from hyperactive.search_config import SearchConfig +from hyperactive.optimizers import HillClimbingOptimizer + +from hyperactive.base import BaseExperiment + + +class SklearnExperiment(BaseExperiment): + def setup(self, estimator, X, y, cv=5): + self.estimator = estimator + self.X = X + self.y = y + self.cv = cv + + def _score(self, params): + model = self.estimator(**params) + scores = cross_val_score(model, self.X, self.y, cv=self.cv) + return scores.mean() + + +data = load_diabetes() +X, y = data.data, data.target + + +search_config = SearchConfig( + max_depth=list(np.arange(2, 15, 1)), + min_samples_split=list(np.arange(2, 25, 2)), +) + +experiment = SklearnExperiment() +experiment.setup(DecisionTreeRegressor, X, y, cv=4) + +optimizer1 = HillClimbingOptimizer() +optimizer1.add_search(experiment, search_config, n_iter=100, n_jobs=2) +optimizer1.run() From 040d63f98786a8eb421b527dfa68840e640da555 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Fri, 14 Feb 2025 09:40:54 +0100 Subject: [PATCH 13/62] get callbacks and catch parameters from experiment --- src/hyperactive/optimizers/_optimizer_api.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index 8d41183c..30bab5b4 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -54,8 +54,6 @@ def add_search( initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, constraints: List[callable] = None, pass_through: Dict = None, - callbacks: Dict[str, callable] = None, - catch: Dict = None, max_score: float = None, early_stopping: Dict = None, random_state: int = None, @@ -89,8 +87,6 @@ def add_search( constraints = constraints or [] pass_through = pass_through or {} - callbacks = callbacks or {} - catch = catch or {} early_stopping = early_stopping or {} search_id = self._default_search_id(search_id, objective_function) @@ -104,8 +100,8 @@ def add_search( initialize=initialize, constraints=constraints, pass_through=pass_through, - callbacks=callbacks, - catch=catch, + callbacks=experiment.callbacks, + catch=experiment.catch, max_score=max_score, early_stopping=early_stopping, random_state=random_state, From e5336f04a6a8da9be4dbaeec37529aa2ec167fe2 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Fri, 14 Feb 2025 10:15:33 +0100 Subject: [PATCH 14/62] set callbacks and catch to empty dict if None --- pyproject.toml | 2 +- src/hyperactive/base/_experiment.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a76457a2..95843bfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "numpy >=1.18.1, <3.0.0", "tqdm >=4.48.0, <5.0.0", "pandas <3.0.0", - "gradient-free-optimizers >=1.2.4, <2.0.0", + "gradient-free-optimizers >=1.5.0, <2.0.0", "scikit-base <1.0.0", ] diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index f0adea77..88375f56 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -14,8 +14,8 @@ def __init__( ): super().__init__() - self.callbacks = callbacks - self.catch = catch + self.callbacks = callbacks or {} + self.catch = catch or {} def __call__(self, **kwargs): """Score parameters, with kwargs call.""" From 1b028c25a8b28b48a663c4a3803da435e94dc840 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Fri, 14 Feb 2025 10:16:33 +0100 Subject: [PATCH 15/62] use experiment to pass objective-function --- src/hyperactive/optimizers/_optimizer_api.py | 10 +++++----- .../optimizers/backend_stuff/print_results.py | 12 +++++++++--- src/hyperactive/optimizers/hyper_optimizer.py | 8 ++++---- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index 30bab5b4..b33d4ed5 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -64,7 +64,7 @@ def add_search( Add a new optimization search process with specified parameters. Parameters: - - objective_function: The objective function to optimize. + - experiment: Experiment class containing the objective-function to optimize. - search_space: Dictionary defining the search space for optimization. - n_iter: Number of iterations for the optimization process. - search_id: Identifier for the search process (default: None). @@ -81,20 +81,20 @@ def add_search( - memory_warm_start: DataFrame containing warm start memory (default: None). """ - objective_function = experiment._score - self.check_list(search_space) constraints = constraints or [] pass_through = pass_through or {} early_stopping = early_stopping or {} - search_id = self._default_search_id(search_id, objective_function) + search_id = self._default_search_id( + search_id, experiment.objective_function + ) s_space = SearchSpace(search_space) self.verbosity = verbosity self.hyper_optimizer.setup_search( - objective_function=objective_function, + experiment=experiment, s_space=s_space, n_iter=n_iter, initialize=initialize, diff --git a/src/hyperactive/optimizers/backend_stuff/print_results.py b/src/hyperactive/optimizers/backend_stuff/print_results.py index e5263e84..195eb29e 100644 --- a/src/hyperactive/optimizers/backend_stuff/print_results.py +++ b/src/hyperactive/optimizers/backend_stuff/print_results.py @@ -86,10 +86,14 @@ def _print_results( if best_additional_results: print(indent, "Best additional results:") add_results_names = list(best_additional_results.keys()) - add_results_names_align = self.align_para_names(add_results_names) + add_results_names_align = self.align_para_names( + add_results_names + ) for best_additional_result in best_additional_results.keys(): - added_spaces = add_results_names_align[best_additional_result] + added_spaces = add_results_names_align[ + best_additional_result + ] print( indent, indent, @@ -123,7 +127,9 @@ def _print_results( def print_process(self, results, nth_process): verbosity = self.verbosity - objective_function = self.opt_pros[nth_process].objective_function + objective_function = self.opt_pros[ + nth_process + ].experiment.objective_function search_space = self.opt_pros[nth_process].s_space.search_space search_data = results["search_data"] diff --git a/src/hyperactive/optimizers/hyper_optimizer.py b/src/hyperactive/optimizers/hyper_optimizer.py index a50d4cba..265cfe17 100644 --- a/src/hyperactive/optimizers/hyper_optimizer.py +++ b/src/hyperactive/optimizers/hyper_optimizer.py @@ -18,7 +18,7 @@ def __init__(self, optimizer_class, opt_params): def setup_search( self, - objective_function, + experiment, s_space, n_iter, initialize, @@ -33,7 +33,7 @@ def setup_search( memory_warm_start, verbosity, ): - self.objective_function = objective_function + self.experiment = experiment self.s_space = s_space self.n_iter = n_iter @@ -113,7 +113,7 @@ def search(self, nth_process, p_bar): self._setup_process(nth_process) gfo_wrapper_model = ObjectiveFunction( - objective_function=self.objective_function, + objective_function=self.experiment.objective_function, callbacks=self.callbacks, catch=self.catch, ) @@ -141,7 +141,7 @@ def search(self, nth_process, p_bar): "[" + str(nth_process) + "] " - + str(self.objective_function.__name__) + + str(self.experiment.__class__.__name__) + " (" + self.optimizer_class.name + ")", From eec01d0ea1436f6614ec8ad24704b202d2e28d5b Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Fri, 14 Feb 2025 10:34:08 +0100 Subject: [PATCH 16/62] show experiment name in output (instead of objective-function) --- .../optimizers/backend_stuff/print_results.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/hyperactive/optimizers/backend_stuff/print_results.py b/src/hyperactive/optimizers/backend_stuff/print_results.py index 195eb29e..b92dbc81 100644 --- a/src/hyperactive/optimizers/backend_stuff/print_results.py +++ b/src/hyperactive/optimizers/backend_stuff/print_results.py @@ -67,14 +67,14 @@ def align_para_names(self, para_names): def _print_results( self, - objective_function, + experiment, best_score, best_para, best_iter, best_additional_results, random_seed, ): - print("\nResults: '{}'".format(objective_function.__name__), " ") + print("\nResults: '{}'".format(experiment.__class__.__name__), " ") if best_para is None: print(indent, "Best score:", best_score, " ") print(indent, "Best parameter set:", best_para, " ") @@ -127,9 +127,7 @@ def _print_results( def print_process(self, results, nth_process): verbosity = self.verbosity - objective_function = self.opt_pros[ - nth_process - ].experiment.objective_function + experiment = self.opt_pros[nth_process].experiment search_space = self.opt_pros[nth_process].s_space.search_space search_data = results["search_data"] @@ -163,7 +161,7 @@ def print_process(self, results, nth_process): if "print_results" in verbosity: self._print_results( - objective_function, + experiment, best_score, best_para, best_iter, From d9f12b64ed631a2b812363e85ddbdbf709610070 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Fri, 14 Feb 2025 10:46:16 +0100 Subject: [PATCH 17/62] get obj-func from experiment --- src/hyperactive/optimizers/backend_stuff/results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperactive/optimizers/backend_stuff/results.py b/src/hyperactive/optimizers/backend_stuff/results.py index 9b014e2b..d92f105f 100644 --- a/src/hyperactive/optimizers/backend_stuff/results.py +++ b/src/hyperactive/optimizers/backend_stuff/results.py @@ -26,7 +26,7 @@ def _sort_results_objFunc(self, objective_function): nth_process = results_["nth_process"] opt = self.opt_pros[nth_process] - objective_function_ = opt.objective_function + objective_function_ = opt.experiment.objective_function search_space_ = opt.s_space() params = list(search_space_.keys()) From ce8b82f9dd40e15a2c4521f9c8110356443da909 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Fri, 14 Feb 2025 10:46:40 +0100 Subject: [PATCH 18/62] readd result attributes --- src/hyperactive/optimizers/_optimizer_api.py | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index b33d4ed5..d77d5acb 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -141,3 +141,52 @@ def run( self.results_ = Results(self.results_list, self.opt_pros) self._print_info() + + def best_para(self, id_): + """ + Retrieve the best parameters for a specific ID from the results. + + Parameters: + - id_ (int): The ID of the parameters to retrieve. + + Returns: + - Union[Dict[str, Union[int, float]], None]: The best parameters for the specified ID if found, otherwise None. + + Raises: + - ValueError: If the objective function name is not recognized. + """ + + return self.results_.best_para(id_) + + def best_score(self, id_): + """ + Return the best score for a specific ID from the results. + + Parameters: + - id_ (int): The ID for which the best score is requested. + """ + + return self.results_.best_score(id_) + + def search_data(self, id_, times=False): + """ + Retrieve search data for a specific ID from the results. Optionally exclude evaluation and iteration times if 'times' is set to False. + + Parameters: + - id_ (int): The ID of the search data to retrieve. + - times (bool, optional): Whether to exclude evaluation and iteration times. Defaults to False. + + Returns: + - pd.DataFrame: The search data for the specified ID. + """ + + search_data_ = self.results_.search_data(id_.objective_function) + + if times == False: + search_data_.drop( + labels=["eval_times", "iter_times"], + axis=1, + inplace=True, + errors="ignore", + ) + return search_data_ From 2e75c3f7fab57540257298d8dd4f20a9a2fc7491 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 23 Feb 2025 09:31:36 +0100 Subject: [PATCH 19/62] remove print --- src/hyperactive/search_config/_properties.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyperactive/search_config/_properties.py b/src/hyperactive/search_config/_properties.py index aa1cf543..5be0e7c6 100644 --- a/src/hyperactive/search_config/_properties.py +++ b/src/hyperactive/search_config/_properties.py @@ -23,6 +23,5 @@ def wrapper(self, *args, **kwargs): self.n_dim = n_dim(self._search_space) self.dim_names = dim_names(self._search_space) self.position_space = position_space(self._search_space) - print(" ---> search-space updated!") return wrapper From 66ca1f65a1fb91335ed9f3786f13e4163b011508 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 23 Feb 2025 09:32:03 +0100 Subject: [PATCH 20/62] reformat --- src/hyperactive/optimizers/backend_stuff/run_search.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/optimizers/backend_stuff/run_search.py b/src/hyperactive/optimizers/backend_stuff/run_search.py index 130e6207..9c9a6a72 100644 --- a/src/hyperactive/optimizers/backend_stuff/run_search.py +++ b/src/hyperactive/optimizers/backend_stuff/run_search.py @@ -50,7 +50,9 @@ def run_search(opt_pros, distribution, n_processes): if n_processes == 1: results_list = single_process(_process_, process_infos) else: - (distribution, process_func), dist_paras = _get_distribution(distribution) + (distribution, process_func), dist_paras = _get_distribution( + distribution + ) results_list = distribution(process_func, process_infos, n_processes) From 957e7de3d00277d7559ce125d3545b4106c8a094 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 23 Feb 2025 09:32:39 +0100 Subject: [PATCH 21/62] add composite optimizer class --- src/hyperactive/composite_optimizer.py | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/hyperactive/composite_optimizer.py diff --git a/src/hyperactive/composite_optimizer.py b/src/hyperactive/composite_optimizer.py new file mode 100644 index 00000000..7cc37a65 --- /dev/null +++ b/src/hyperactive/composite_optimizer.py @@ -0,0 +1,44 @@ +from typing import Union +from .optimizers.backend_stuff.run_search import run_search +from .optimizers.backend_stuff.results import Results +from .optimizers.backend_stuff.print_results import PrintResults + + +class CompositeOptimizer: + def __init__(self, *optimizers): + self.optimizers = optimizers + + def run( + self, + max_time=None, + distribution: str = "multiprocessing", + n_processes: Union[str, int] = "auto", + verbosity: list = ["progress_bar", "print_results", "print_times"], + ): + self.verbosity = verbosity + + self.all_opt_pros = {} + for optimizer in self.optimizers: + self.all_opt_pros.update(optimizer.opt_pros) + + for opt in self.all_opt_pros.values(): + opt.max_time = max_time + + self.results_list = run_search( + self.all_opt_pros, distribution, n_processes + ) + + self.results_ = Results(self.results_list, self.all_opt_pros) + + self._print_info() + + def _print_info(self): + print_res = PrintResults(self.all_opt_pros, self.verbosity) + + if self.verbosity: + for _ in range(len(self.all_opt_pros)): + print("") + + for results in self.results_list: + nth_process = results["nth_process"] + print_res.print_process(results, nth_process) From 549e20013694389036f41d28959142bb0d3e5fdf Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 23 Feb 2025 09:33:08 +0100 Subject: [PATCH 22/62] use comp-opt class to enable parallel via 'add'-method --- src/hyperactive/optimizers/_optimizer_api.py | 44 +++++++------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index d77d5acb..618db0c6 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -1,16 +1,14 @@ """Base class for optimizer.""" -import numpy as np -from typing import Union, List, Dict, Type -import copy +from typing import Union, List, Dict import multiprocessing as mp import pandas as pd from .backend_stuff.search_space import SearchSpace -from .backend_stuff.run_search import run_search from .hyper_optimizer import HyperOptimizer -from .backend_stuff.results import Results -from .backend_stuff.print_results import PrintResults + + +from ..composite_optimizer import CompositeOptimizer from skbase.base import BaseObject @@ -18,13 +16,15 @@ class BaseOptimizer(BaseObject): """Base class for optimizer.""" - opt_pros = {} + opt_pros: dict def __init__(self, optimizer_class, opt_params): super().__init__() self.opt_params = opt_params self.hyper_optimizer = HyperOptimizer(optimizer_class, opt_params) + self.opt_pros = {} + @staticmethod def _default_search_id(search_id, objective_function): if not search_id: @@ -116,16 +116,8 @@ def add_search( nth_process = len(self.opt_pros) self.opt_pros[nth_process] = self.hyper_optimizer - def _print_info(self): - print_res = PrintResults(self.opt_pros, self.verbosity) - - if self.verbosity: - for _ in range(len(self.opt_pros)): - print("") - - for results in self.results_list: - nth_process = results["nth_process"] - print_res.print_process(results, nth_process) + def __add__(self, optimizer_instance): + return CompositeOptimizer(self, optimizer_instance) def run( self, @@ -133,14 +125,8 @@ def run( distribution: str = "multiprocessing", n_processes: Union[str, int] = "auto", ): - for opt in self.opt_pros.values(): - opt.max_time = max_time - - self.results_list = run_search(self.opt_pros, distribution, n_processes) - - self.results_ = Results(self.results_list, self.opt_pros) - - self._print_info() + self.comp_opt = CompositeOptimizer(self) + self.comp_opt.run(max_time, distribution, n_processes, self.verbosity) def best_para(self, id_): """ @@ -156,7 +142,7 @@ def best_para(self, id_): - ValueError: If the objective function name is not recognized. """ - return self.results_.best_para(id_) + return self.comp_opt.results_.best_para(id_) def best_score(self, id_): """ @@ -166,7 +152,7 @@ def best_score(self, id_): - id_ (int): The ID for which the best score is requested. """ - return self.results_.best_score(id_) + return self.comp_opt.results_.best_score(id_) def search_data(self, id_, times=False): """ @@ -180,7 +166,9 @@ def search_data(self, id_, times=False): - pd.DataFrame: The search data for the specified ID. """ - search_data_ = self.results_.search_data(id_.objective_function) + search_data_ = self.comp_opt.results_.search_data( + id_.objective_function + ) if times == False: search_data_.drop( From 7b8813cb7cfb8dd0404fb3214e5051692192463e Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 23 Feb 2025 09:44:03 +0100 Subject: [PATCH 23/62] enable adding more than 2 optimizers together --- src/hyperactive/composite_optimizer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/composite_optimizer.py b/src/hyperactive/composite_optimizer.py index 7cc37a65..ef3f98a5 100644 --- a/src/hyperactive/composite_optimizer.py +++ b/src/hyperactive/composite_optimizer.py @@ -6,7 +6,11 @@ class CompositeOptimizer: def __init__(self, *optimizers): - self.optimizers = optimizers + self.optimizers = list(optimizers) + + def __add__(self, optimizer_instance): + self.optimizers.append(optimizer_instance) + return self def run( self, From e110061a1f8344d8fc522891037f9f6953b343c0 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 23 Feb 2025 17:45:31 +0100 Subject: [PATCH 24/62] rename hyper_optimizer -> search and fix multiprocessing --- src/hyperactive/composite_optimizer.py | 18 ++++--- src/hyperactive/optimizers/_optimizer_api.py | 53 +++++++++++-------- .../optimizers/backend_stuff/distribution.py | 2 + .../optimizers/backend_stuff/process.py | 8 +-- .../optimizers/backend_stuff/run_search.py | 16 +++--- .../{hyper_optimizer.py => search.py} | 25 +++++---- 6 files changed, 67 insertions(+), 55 deletions(-) rename src/hyperactive/optimizers/{hyper_optimizer.py => search.py} (93%) diff --git a/src/hyperactive/composite_optimizer.py b/src/hyperactive/composite_optimizer.py index ef3f98a5..d934c119 100644 --- a/src/hyperactive/composite_optimizer.py +++ b/src/hyperactive/composite_optimizer.py @@ -5,6 +5,8 @@ class CompositeOptimizer: + optimizers: list + def __init__(self, *optimizers): self.optimizers = list(optimizers) @@ -21,26 +23,26 @@ def run( ): self.verbosity = verbosity - self.all_opt_pros = {} + self.collected_searches = [] for optimizer in self.optimizers: - self.all_opt_pros.update(optimizer.opt_pros) + self.collected_searches += optimizer.searches - for opt in self.all_opt_pros.values(): - opt.max_time = max_time + for nth_process, search in enumerate(self.collected_searches): + search.pass_args(max_time, nth_process) self.results_list = run_search( - self.all_opt_pros, distribution, n_processes + self.collected_searches, distribution, n_processes ) - self.results_ = Results(self.results_list, self.all_opt_pros) + self.results_ = Results(self.results_list, self.collected_searches) self._print_info() def _print_info(self): - print_res = PrintResults(self.all_opt_pros, self.verbosity) + print_res = PrintResults(self.collected_searches, self.verbosity) if self.verbosity: - for _ in range(len(self.all_opt_pros)): + for _ in range(len(self.collected_searches)): print("") for results in self.results_list: diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index 618db0c6..17db4908 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -5,7 +5,7 @@ import pandas as pd from .backend_stuff.search_space import SearchSpace -from .hyper_optimizer import HyperOptimizer +from .search import Search from ..composite_optimizer import CompositeOptimizer @@ -16,14 +16,18 @@ class BaseOptimizer(BaseObject): """Base class for optimizer.""" + n_search: int + searches: list opt_pros: dict def __init__(self, optimizer_class, opt_params): super().__init__() + + self.optimizer_class = optimizer_class self.opt_params = opt_params - self.hyper_optimizer = HyperOptimizer(optimizer_class, opt_params) - self.opt_pros = {} + self.n_search = 0 + self.searches = [] @staticmethod def _default_search_id(search_id, objective_function): @@ -81,6 +85,8 @@ def add_search( - memory_warm_start: DataFrame containing warm start memory (default: None). """ + self.n_search += 1 + self.check_list(search_space) constraints = constraints or [] @@ -93,28 +99,31 @@ def add_search( s_space = SearchSpace(search_space) self.verbosity = verbosity - self.hyper_optimizer.setup_search( - experiment=experiment, - s_space=s_space, - n_iter=n_iter, - initialize=initialize, - constraints=constraints, - pass_through=pass_through, - callbacks=experiment.callbacks, - catch=experiment.catch, - max_score=max_score, - early_stopping=early_stopping, - random_state=random_state, - memory=memory, - memory_warm_start=memory_warm_start, - verbosity=verbosity, - ) - n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs for _ in range(n_jobs): - nth_process = len(self.opt_pros) - self.opt_pros[nth_process] = self.hyper_optimizer + search = Search(self.optimizer_class, self.opt_params) + search.setup( + experiment=experiment, + s_space=s_space, + n_iter=n_iter, + initialize=initialize, + constraints=constraints, + pass_through=pass_through, + callbacks=experiment.callbacks, + catch=experiment.catch, + max_score=max_score, + early_stopping=early_stopping, + random_state=random_state, + memory=memory, + memory_warm_start=memory_warm_start, + verbosity=verbosity, + ) + self.searches.append(search) + + @property + def nth_search(self): + return len(self.composite_opt.optimizers) def __add__(self, optimizer_instance): return CompositeOptimizer(self, optimizer_instance) diff --git a/src/hyperactive/optimizers/backend_stuff/distribution.py b/src/hyperactive/optimizers/backend_stuff/distribution.py index eea37214..4770aaad 100644 --- a/src/hyperactive/optimizers/backend_stuff/distribution.py +++ b/src/hyperactive/optimizers/backend_stuff/distribution.py @@ -20,6 +20,8 @@ def single_process(process_func, process_infos): def multiprocessing_wrapper(process_func, process_infos, n_processes): import multiprocessing as mp + process_infos = tuple(process_infos) + with mp.Pool( n_processes, initializer=initializer, initargs=initargs ) as pool: diff --git a/src/hyperactive/optimizers/backend_stuff/process.py b/src/hyperactive/optimizers/backend_stuff/process.py index f008dc80..3113743f 100644 --- a/src/hyperactive/optimizers/backend_stuff/process.py +++ b/src/hyperactive/optimizers/backend_stuff/process.py @@ -6,10 +6,10 @@ from tqdm import tqdm -def _process_(nth_process, optimizer): +def _process_(optimizer): if "progress_bar" in optimizer.verbosity: p_bar = tqdm( - position=nth_process, + position=optimizer.nth_process, total=optimizer.n_iter, ascii=" ─", colour="Yellow", @@ -17,7 +17,7 @@ def _process_(nth_process, optimizer): else: p_bar = None - optimizer.search(nth_process, p_bar) + optimizer._search(p_bar) if p_bar: p_bar.colour = "GREEN" @@ -25,7 +25,7 @@ def _process_(nth_process, optimizer): p_bar.close() return { - "nth_process": nth_process, + "nth_process": optimizer.nth_process, "best_para": optimizer.best_para, "best_score": optimizer.best_score, "best_iter": optimizer.best_since_iter, diff --git a/src/hyperactive/optimizers/backend_stuff/run_search.py b/src/hyperactive/optimizers/backend_stuff/run_search.py index 9c9a6a72..c9fd13e1 100644 --- a/src/hyperactive/optimizers/backend_stuff/run_search.py +++ b/src/hyperactive/optimizers/backend_stuff/run_search.py @@ -37,23 +37,19 @@ def _get_distribution(distribution): return dist_dict[distribution], {} -def run_search(opt_pros, distribution, n_processes): - opts = list(opt_pros.values()) - - processes = list(opt_pros.keys()) - optimizers = list(opt_pros.values()) - process_infos = list(zip(processes, optimizers)) - +def run_search(searches, distribution, n_processes): if n_processes == "auto": - n_processes = len(process_infos) + n_processes = len(searches) + + searches_tuple = [(search,) for search in searches] if n_processes == 1: - results_list = single_process(_process_, process_infos) + results_list = single_process(_process_, searches) else: (distribution, process_func), dist_paras = _get_distribution( distribution ) - results_list = distribution(process_func, process_infos, n_processes) + results_list = distribution(process_func, searches_tuple, n_processes) return results_list diff --git a/src/hyperactive/optimizers/hyper_optimizer.py b/src/hyperactive/optimizers/search.py similarity index 93% rename from src/hyperactive/optimizers/hyper_optimizer.py rename to src/hyperactive/optimizers/search.py index 265cfe17..936fa31d 100644 --- a/src/hyperactive/optimizers/hyper_optimizer.py +++ b/src/hyperactive/optimizers/search.py @@ -2,21 +2,22 @@ # Email: simon.blanke@yahoo.com # License: MIT License -import numpy as np - from .objective_function import ObjectiveFunction from .hyper_gradient_conv import HyperGradientConv from .optimizer_attributes import OptimizerAttributes from .constraint import Constraint -class HyperOptimizer(OptimizerAttributes): +class Search(OptimizerAttributes): + max_time: float + nth_process: int + def __init__(self, optimizer_class, opt_params): super().__init__() self.optimizer_class = optimizer_class self.opt_params = opt_params - def setup_search( + def setup( self, experiment, s_space, @@ -54,6 +55,10 @@ def setup_search( else: self.verbosity = [] + def pass_args(self, max_time, nth_process): + self.max_time = max_time + self.nth_process = nth_process + def convert_results2hyper(self): self.eval_times = sum(self.gfo_optimizer.eval_times) self.iter_times = sum(self.gfo_optimizer.iter_times) @@ -77,9 +82,7 @@ def convert_results2hyper(self): self.s_space.dim_keys + ["score"] ].reset_index(drop=True) - def _setup_process(self, nth_process): - self.nth_process = nth_process - + def _setup_process(self): self.hg_conv = HyperGradientConv(self.s_space) initialize = self.hg_conv.conv_initialize(self.initialize) @@ -103,14 +106,14 @@ def _setup_process(self, nth_process): initialize=initialize, constraints=gfo_constraints, random_state=self.random_state, - nth_process=nth_process, + nth_process=self.nth_process, **self.opt_params, ) self.conv = self.gfo_optimizer.conv - def search(self, nth_process, p_bar): - self._setup_process(nth_process) + def _search(self, p_bar): + self._setup_process() gfo_wrapper_model = ObjectiveFunction( objective_function=self.experiment.objective_function, @@ -139,7 +142,7 @@ def search(self, nth_process, p_bar): if p_bar: p_bar.set_description( "[" - + str(nth_process) + + str(self.nth_process) + "] " + str(self.experiment.__class__.__name__) + " (" From a5655497072b397c8de785e1bdcadbefe1ac981f Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Tue, 4 Mar 2025 10:50:13 +0100 Subject: [PATCH 25/62] refactor callbacks/catch-parameter --- src/hyperactive/optimizers/_optimizer_api.py | 2 -- src/hyperactive/optimizers/objective_function.py | 8 ++++---- src/hyperactive/optimizers/search.py | 8 +------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index 17db4908..f7dcdb1b 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -110,8 +110,6 @@ def add_search( initialize=initialize, constraints=constraints, pass_through=pass_through, - callbacks=experiment.callbacks, - catch=experiment.catch, max_score=max_score, early_stopping=early_stopping, random_state=random_state, diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/objective_function.py index 6995e640..a3ecfced 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/objective_function.py @@ -16,12 +16,12 @@ def gfo2hyper(search_space, para): class ObjectiveFunction(DictClass): - def __init__(self, objective_function, callbacks, catch): + def __init__(self, experiment): super().__init__() - self.objective_function = objective_function - self.callbacks = callbacks - self.catch = catch + self.objective_function = experiment.objective_function + self.callbacks = experiment.callbacks + self.catch = experiment.catch self.nth_iter = 0 diff --git a/src/hyperactive/optimizers/search.py b/src/hyperactive/optimizers/search.py index 936fa31d..d51e5bf7 100644 --- a/src/hyperactive/optimizers/search.py +++ b/src/hyperactive/optimizers/search.py @@ -25,8 +25,6 @@ def setup( initialize, constraints, pass_through, - callbacks, - catch, max_score, early_stopping, random_state, @@ -41,8 +39,6 @@ def setup( self.initialize = initialize self.constraints = constraints self.pass_through = pass_through - self.callbacks = callbacks - self.catch = catch self.max_score = max_score self.early_stopping = early_stopping self.random_state = random_state @@ -116,9 +112,7 @@ def _search(self, p_bar): self._setup_process() gfo_wrapper_model = ObjectiveFunction( - objective_function=self.experiment.objective_function, - callbacks=self.callbacks, - catch=self.catch, + experiment=self.experiment, ) gfo_wrapper_model.pass_through = self.pass_through From b83143f0e3228e8dfd2615267137b6c073db894d Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 16 Mar 2025 18:30:33 +0100 Subject: [PATCH 26/62] move old tests; add new test-base file --- tests/test_base.py | 22 +++++++++++++++++++ tests/test_setup/__init__.py | 2 ++ tests/test_setup/experiment.py | 11 ++++++++++ tests/test_setup/search_config.py | 8 +++++++ .../__init__.py | 0 .../__init__.py | 0 .../_parametrize.py | 0 .../_test_memory_warm_start.py | 0 .../_test_memory_warm_start_smbo.py | 0 .../_test_strategy_combinations.py | 0 .../_test_strategy_multiprocessing.py | 0 .../_local_test_timings}/__init__.py | 0 .../_local_test_timings/_search_space_list.py | 0 .../_local_test_timings/_test_memory.py | 0 .../_test_memory_warm_start.py | 0 .../_test_memory_warm_start_n_jobs.py | 0 .../_test_shared_memory.py | 0 .../_local_test_timings/_test_warm_start.py | 0 .../_test_warm_start_n_jobs.py | 0 {tests => tests_old}/_test_examples.py | 0 .../integrations}/__init__.py | 0 .../integrations/sklearn}/__init__.py | 0 .../sklearn/test_parametrize_with_checks.py | 0 .../integrations/sklearn/test_sklearn_api.py | 0 {tests => tests_old}/test_callbacks.py | 0 {tests => tests_old}/test_catch.py | 0 {tests => tests_old}/test_constr_opt.py | 0 {tests => tests_old}/test_distribution.py | 0 {tests => tests_old}/test_early_stop.py | 0 .../test_empty_output}/__init__.py | 0 .../test_empty_output/non_verbose.py | 0 .../test_empty_output/test_empty_output.py | 0 .../test_empty_output/verbose.py | 0 .../test_hyper_gradient_trafo.py | 0 {tests => tests_old}/test_initializers.py | 0 .../test_issues}/__init__.py | 0 .../test_issues/test_issue_25.py | 0 .../test_issues/test_issue_29.py | 0 .../test_issues/test_issue_34.py | 0 {tests => tests_old}/test_max_score.py | 0 {tests => tests_old}/test_max_time.py | 0 {tests => tests_old}/test_obj_func_arg.py | 0 .../test_optimization_strategies}/__init__.py | 0 .../_parametrize.py | 0 .../test_constr_opt.py | 0 .../test_early_stopping.py | 0 .../test_search_space_pruning.py | 0 .../test_optimizers}/__init__.py | 0 .../test_optimizers/_parametrize.py | 0 .../test_optimizers/test_best_results.py | 0 .../test_optimizers/test_gfo_wrapper.py | 0 .../test_optimizers/test_memory.py | 0 .../test_optimization_strategies.py | 0 {tests => tests_old}/test_pass_through.py | 0 {tests => tests_old}/test_random_state.py | 0 {tests => tests_old}/test_results.py | 0 {tests => tests_old}/test_results_methods.py | 0 {tests => tests_old}/test_search_spaces.py | 0 tests_old/test_warm_starts/__init__.py | 0 .../test_memory_warm_start.py | 0 .../test_warm_starts/test_warm_start.py | 0 .../test_warm_starts/test_warm_start_smbo.py | 0 62 files changed, 43 insertions(+) create mode 100644 tests/test_base.py create mode 100644 tests/test_setup/__init__.py create mode 100644 tests/test_setup/experiment.py create mode 100644 tests/test_setup/search_config.py rename {tests/_local_test_optimization_strategies => tests_old}/__init__.py (100%) rename {tests/_local_test_timings => tests_old/_local_test_optimization_strategies}/__init__.py (100%) rename {tests => tests_old}/_local_test_optimization_strategies/_parametrize.py (100%) rename {tests => tests_old}/_local_test_optimization_strategies/_test_memory_warm_start.py (100%) rename {tests => tests_old}/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py (100%) rename {tests => tests_old}/_local_test_optimization_strategies/_test_strategy_combinations.py (100%) rename {tests => tests_old}/_local_test_optimization_strategies/_test_strategy_multiprocessing.py (100%) rename {tests/integrations => tests_old/_local_test_timings}/__init__.py (100%) rename {tests => tests_old}/_local_test_timings/_search_space_list.py (100%) rename {tests => tests_old}/_local_test_timings/_test_memory.py (100%) rename {tests => tests_old}/_local_test_timings/_test_memory_warm_start.py (100%) rename {tests => tests_old}/_local_test_timings/_test_memory_warm_start_n_jobs.py (100%) rename {tests => tests_old}/_local_test_timings/_test_shared_memory.py (100%) rename {tests => tests_old}/_local_test_timings/_test_warm_start.py (100%) rename {tests => tests_old}/_local_test_timings/_test_warm_start_n_jobs.py (100%) rename {tests => tests_old}/_test_examples.py (100%) rename {tests/integrations/sklearn => tests_old/integrations}/__init__.py (100%) rename {tests/test_empty_output => tests_old/integrations/sklearn}/__init__.py (100%) rename {tests => tests_old}/integrations/sklearn/test_parametrize_with_checks.py (100%) rename {tests => tests_old}/integrations/sklearn/test_sklearn_api.py (100%) rename {tests => tests_old}/test_callbacks.py (100%) rename {tests => tests_old}/test_catch.py (100%) rename {tests => tests_old}/test_constr_opt.py (100%) rename {tests => tests_old}/test_distribution.py (100%) rename {tests => tests_old}/test_early_stop.py (100%) rename {tests/test_issues => tests_old/test_empty_output}/__init__.py (100%) rename {tests => tests_old}/test_empty_output/non_verbose.py (100%) rename {tests => tests_old}/test_empty_output/test_empty_output.py (100%) rename {tests => tests_old}/test_empty_output/verbose.py (100%) rename {tests => tests_old}/test_hyper_gradient_trafo.py (100%) rename {tests => tests_old}/test_initializers.py (100%) rename {tests/test_optimization_strategies => tests_old/test_issues}/__init__.py (100%) rename {tests => tests_old}/test_issues/test_issue_25.py (100%) rename {tests => tests_old}/test_issues/test_issue_29.py (100%) rename {tests => tests_old}/test_issues/test_issue_34.py (100%) rename {tests => tests_old}/test_max_score.py (100%) rename {tests => tests_old}/test_max_time.py (100%) rename {tests => tests_old}/test_obj_func_arg.py (100%) rename {tests/test_optimizers => tests_old/test_optimization_strategies}/__init__.py (100%) rename {tests => tests_old}/test_optimization_strategies/_parametrize.py (100%) rename {tests => tests_old}/test_optimization_strategies/test_constr_opt.py (100%) rename {tests => tests_old}/test_optimization_strategies/test_early_stopping.py (100%) rename {tests => tests_old}/test_optimization_strategies/test_search_space_pruning.py (100%) rename {tests/test_warm_starts => tests_old/test_optimizers}/__init__.py (100%) rename {tests => tests_old}/test_optimizers/_parametrize.py (100%) rename {tests => tests_old}/test_optimizers/test_best_results.py (100%) rename {tests => tests_old}/test_optimizers/test_gfo_wrapper.py (100%) rename {tests => tests_old}/test_optimizers/test_memory.py (100%) rename {tests => tests_old}/test_optimizers/test_optimization_strategies.py (100%) rename {tests => tests_old}/test_pass_through.py (100%) rename {tests => tests_old}/test_random_state.py (100%) rename {tests => tests_old}/test_results.py (100%) rename {tests => tests_old}/test_results_methods.py (100%) rename {tests => tests_old}/test_search_spaces.py (100%) create mode 100644 tests_old/test_warm_starts/__init__.py rename {tests => tests_old}/test_warm_starts/test_memory_warm_start.py (100%) rename {tests => tests_old}/test_warm_starts/test_warm_start.py (100%) rename {tests => tests_old}/test_warm_starts/test_warm_start_smbo.py (100%) diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 00000000..8819d824 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,22 @@ +import copy +import pytest +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer + + +from .test_setup import SphereFunction, search_config + + +objective_function = SphereFunction() + + +def test_callback_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + objective_function, + search_config, + n_iter=100, + ) + hyper.run() diff --git a/tests/test_setup/__init__.py b/tests/test_setup/__init__.py new file mode 100644 index 00000000..497df055 --- /dev/null +++ b/tests/test_setup/__init__.py @@ -0,0 +1,2 @@ +from .experiment import SphereFunction +from .search_config import search_config \ No newline at end of file diff --git a/tests/test_setup/experiment.py b/tests/test_setup/experiment.py new file mode 100644 index 00000000..bea4deb7 --- /dev/null +++ b/tests/test_setup/experiment.py @@ -0,0 +1,11 @@ +import numpy as np + +from hyperactive.base import BaseExperiment + + +class SphereFunction(BaseExperiment): + def setup(self, n_dim=2): + self.n_dim = n_dim + + def objective_function(self, params): + return np.sum(params["x0"] ** 2 + params["x1"] ** 2) \ No newline at end of file diff --git a/tests/test_setup/search_config.py b/tests/test_setup/search_config.py new file mode 100644 index 00000000..03e7e725 --- /dev/null +++ b/tests/test_setup/search_config.py @@ -0,0 +1,8 @@ +import numpy as np +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x0=list(np.arange(2, 15, 1)), + x1=list(np.arange(2, 25, 2)), +) diff --git a/tests/_local_test_optimization_strategies/__init__.py b/tests_old/__init__.py similarity index 100% rename from tests/_local_test_optimization_strategies/__init__.py rename to tests_old/__init__.py diff --git a/tests/_local_test_timings/__init__.py b/tests_old/_local_test_optimization_strategies/__init__.py similarity index 100% rename from tests/_local_test_timings/__init__.py rename to tests_old/_local_test_optimization_strategies/__init__.py diff --git a/tests/_local_test_optimization_strategies/_parametrize.py b/tests_old/_local_test_optimization_strategies/_parametrize.py similarity index 100% rename from tests/_local_test_optimization_strategies/_parametrize.py rename to tests_old/_local_test_optimization_strategies/_parametrize.py diff --git a/tests/_local_test_optimization_strategies/_test_memory_warm_start.py b/tests_old/_local_test_optimization_strategies/_test_memory_warm_start.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_memory_warm_start.py rename to tests_old/_local_test_optimization_strategies/_test_memory_warm_start.py diff --git a/tests/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py b/tests_old/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py rename to tests_old/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py diff --git a/tests/_local_test_optimization_strategies/_test_strategy_combinations.py b/tests_old/_local_test_optimization_strategies/_test_strategy_combinations.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_strategy_combinations.py rename to tests_old/_local_test_optimization_strategies/_test_strategy_combinations.py diff --git a/tests/_local_test_optimization_strategies/_test_strategy_multiprocessing.py b/tests_old/_local_test_optimization_strategies/_test_strategy_multiprocessing.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_strategy_multiprocessing.py rename to tests_old/_local_test_optimization_strategies/_test_strategy_multiprocessing.py diff --git a/tests/integrations/__init__.py b/tests_old/_local_test_timings/__init__.py similarity index 100% rename from tests/integrations/__init__.py rename to tests_old/_local_test_timings/__init__.py diff --git a/tests/_local_test_timings/_search_space_list.py b/tests_old/_local_test_timings/_search_space_list.py similarity index 100% rename from tests/_local_test_timings/_search_space_list.py rename to tests_old/_local_test_timings/_search_space_list.py diff --git a/tests/_local_test_timings/_test_memory.py b/tests_old/_local_test_timings/_test_memory.py similarity index 100% rename from tests/_local_test_timings/_test_memory.py rename to tests_old/_local_test_timings/_test_memory.py diff --git a/tests/_local_test_timings/_test_memory_warm_start.py b/tests_old/_local_test_timings/_test_memory_warm_start.py similarity index 100% rename from tests/_local_test_timings/_test_memory_warm_start.py rename to tests_old/_local_test_timings/_test_memory_warm_start.py diff --git a/tests/_local_test_timings/_test_memory_warm_start_n_jobs.py b/tests_old/_local_test_timings/_test_memory_warm_start_n_jobs.py similarity index 100% rename from tests/_local_test_timings/_test_memory_warm_start_n_jobs.py rename to tests_old/_local_test_timings/_test_memory_warm_start_n_jobs.py diff --git a/tests/_local_test_timings/_test_shared_memory.py b/tests_old/_local_test_timings/_test_shared_memory.py similarity index 100% rename from tests/_local_test_timings/_test_shared_memory.py rename to tests_old/_local_test_timings/_test_shared_memory.py diff --git a/tests/_local_test_timings/_test_warm_start.py b/tests_old/_local_test_timings/_test_warm_start.py similarity index 100% rename from tests/_local_test_timings/_test_warm_start.py rename to tests_old/_local_test_timings/_test_warm_start.py diff --git a/tests/_local_test_timings/_test_warm_start_n_jobs.py b/tests_old/_local_test_timings/_test_warm_start_n_jobs.py similarity index 100% rename from tests/_local_test_timings/_test_warm_start_n_jobs.py rename to tests_old/_local_test_timings/_test_warm_start_n_jobs.py diff --git a/tests/_test_examples.py b/tests_old/_test_examples.py similarity index 100% rename from tests/_test_examples.py rename to tests_old/_test_examples.py diff --git a/tests/integrations/sklearn/__init__.py b/tests_old/integrations/__init__.py similarity index 100% rename from tests/integrations/sklearn/__init__.py rename to tests_old/integrations/__init__.py diff --git a/tests/test_empty_output/__init__.py b/tests_old/integrations/sklearn/__init__.py similarity index 100% rename from tests/test_empty_output/__init__.py rename to tests_old/integrations/sklearn/__init__.py diff --git a/tests/integrations/sklearn/test_parametrize_with_checks.py b/tests_old/integrations/sklearn/test_parametrize_with_checks.py similarity index 100% rename from tests/integrations/sklearn/test_parametrize_with_checks.py rename to tests_old/integrations/sklearn/test_parametrize_with_checks.py diff --git a/tests/integrations/sklearn/test_sklearn_api.py b/tests_old/integrations/sklearn/test_sklearn_api.py similarity index 100% rename from tests/integrations/sklearn/test_sklearn_api.py rename to tests_old/integrations/sklearn/test_sklearn_api.py diff --git a/tests/test_callbacks.py b/tests_old/test_callbacks.py similarity index 100% rename from tests/test_callbacks.py rename to tests_old/test_callbacks.py diff --git a/tests/test_catch.py b/tests_old/test_catch.py similarity index 100% rename from tests/test_catch.py rename to tests_old/test_catch.py diff --git a/tests/test_constr_opt.py b/tests_old/test_constr_opt.py similarity index 100% rename from tests/test_constr_opt.py rename to tests_old/test_constr_opt.py diff --git a/tests/test_distribution.py b/tests_old/test_distribution.py similarity index 100% rename from tests/test_distribution.py rename to tests_old/test_distribution.py diff --git a/tests/test_early_stop.py b/tests_old/test_early_stop.py similarity index 100% rename from tests/test_early_stop.py rename to tests_old/test_early_stop.py diff --git a/tests/test_issues/__init__.py b/tests_old/test_empty_output/__init__.py similarity index 100% rename from tests/test_issues/__init__.py rename to tests_old/test_empty_output/__init__.py diff --git a/tests/test_empty_output/non_verbose.py b/tests_old/test_empty_output/non_verbose.py similarity index 100% rename from tests/test_empty_output/non_verbose.py rename to tests_old/test_empty_output/non_verbose.py diff --git a/tests/test_empty_output/test_empty_output.py b/tests_old/test_empty_output/test_empty_output.py similarity index 100% rename from tests/test_empty_output/test_empty_output.py rename to tests_old/test_empty_output/test_empty_output.py diff --git a/tests/test_empty_output/verbose.py b/tests_old/test_empty_output/verbose.py similarity index 100% rename from tests/test_empty_output/verbose.py rename to tests_old/test_empty_output/verbose.py diff --git a/tests/test_hyper_gradient_trafo.py b/tests_old/test_hyper_gradient_trafo.py similarity index 100% rename from tests/test_hyper_gradient_trafo.py rename to tests_old/test_hyper_gradient_trafo.py diff --git a/tests/test_initializers.py b/tests_old/test_initializers.py similarity index 100% rename from tests/test_initializers.py rename to tests_old/test_initializers.py diff --git a/tests/test_optimization_strategies/__init__.py b/tests_old/test_issues/__init__.py similarity index 100% rename from tests/test_optimization_strategies/__init__.py rename to tests_old/test_issues/__init__.py diff --git a/tests/test_issues/test_issue_25.py b/tests_old/test_issues/test_issue_25.py similarity index 100% rename from tests/test_issues/test_issue_25.py rename to tests_old/test_issues/test_issue_25.py diff --git a/tests/test_issues/test_issue_29.py b/tests_old/test_issues/test_issue_29.py similarity index 100% rename from tests/test_issues/test_issue_29.py rename to tests_old/test_issues/test_issue_29.py diff --git a/tests/test_issues/test_issue_34.py b/tests_old/test_issues/test_issue_34.py similarity index 100% rename from tests/test_issues/test_issue_34.py rename to tests_old/test_issues/test_issue_34.py diff --git a/tests/test_max_score.py b/tests_old/test_max_score.py similarity index 100% rename from tests/test_max_score.py rename to tests_old/test_max_score.py diff --git a/tests/test_max_time.py b/tests_old/test_max_time.py similarity index 100% rename from tests/test_max_time.py rename to tests_old/test_max_time.py diff --git a/tests/test_obj_func_arg.py b/tests_old/test_obj_func_arg.py similarity index 100% rename from tests/test_obj_func_arg.py rename to tests_old/test_obj_func_arg.py diff --git a/tests/test_optimizers/__init__.py b/tests_old/test_optimization_strategies/__init__.py similarity index 100% rename from tests/test_optimizers/__init__.py rename to tests_old/test_optimization_strategies/__init__.py diff --git a/tests/test_optimization_strategies/_parametrize.py b/tests_old/test_optimization_strategies/_parametrize.py similarity index 100% rename from tests/test_optimization_strategies/_parametrize.py rename to tests_old/test_optimization_strategies/_parametrize.py diff --git a/tests/test_optimization_strategies/test_constr_opt.py b/tests_old/test_optimization_strategies/test_constr_opt.py similarity index 100% rename from tests/test_optimization_strategies/test_constr_opt.py rename to tests_old/test_optimization_strategies/test_constr_opt.py diff --git a/tests/test_optimization_strategies/test_early_stopping.py b/tests_old/test_optimization_strategies/test_early_stopping.py similarity index 100% rename from tests/test_optimization_strategies/test_early_stopping.py rename to tests_old/test_optimization_strategies/test_early_stopping.py diff --git a/tests/test_optimization_strategies/test_search_space_pruning.py b/tests_old/test_optimization_strategies/test_search_space_pruning.py similarity index 100% rename from tests/test_optimization_strategies/test_search_space_pruning.py rename to tests_old/test_optimization_strategies/test_search_space_pruning.py diff --git a/tests/test_warm_starts/__init__.py b/tests_old/test_optimizers/__init__.py similarity index 100% rename from tests/test_warm_starts/__init__.py rename to tests_old/test_optimizers/__init__.py diff --git a/tests/test_optimizers/_parametrize.py b/tests_old/test_optimizers/_parametrize.py similarity index 100% rename from tests/test_optimizers/_parametrize.py rename to tests_old/test_optimizers/_parametrize.py diff --git a/tests/test_optimizers/test_best_results.py b/tests_old/test_optimizers/test_best_results.py similarity index 100% rename from tests/test_optimizers/test_best_results.py rename to tests_old/test_optimizers/test_best_results.py diff --git a/tests/test_optimizers/test_gfo_wrapper.py b/tests_old/test_optimizers/test_gfo_wrapper.py similarity index 100% rename from tests/test_optimizers/test_gfo_wrapper.py rename to tests_old/test_optimizers/test_gfo_wrapper.py diff --git a/tests/test_optimizers/test_memory.py b/tests_old/test_optimizers/test_memory.py similarity index 100% rename from tests/test_optimizers/test_memory.py rename to tests_old/test_optimizers/test_memory.py diff --git a/tests/test_optimizers/test_optimization_strategies.py b/tests_old/test_optimizers/test_optimization_strategies.py similarity index 100% rename from tests/test_optimizers/test_optimization_strategies.py rename to tests_old/test_optimizers/test_optimization_strategies.py diff --git a/tests/test_pass_through.py b/tests_old/test_pass_through.py similarity index 100% rename from tests/test_pass_through.py rename to tests_old/test_pass_through.py diff --git a/tests/test_random_state.py b/tests_old/test_random_state.py similarity index 100% rename from tests/test_random_state.py rename to tests_old/test_random_state.py diff --git a/tests/test_results.py b/tests_old/test_results.py similarity index 100% rename from tests/test_results.py rename to tests_old/test_results.py diff --git a/tests/test_results_methods.py b/tests_old/test_results_methods.py similarity index 100% rename from tests/test_results_methods.py rename to tests_old/test_results_methods.py diff --git a/tests/test_search_spaces.py b/tests_old/test_search_spaces.py similarity index 100% rename from tests/test_search_spaces.py rename to tests_old/test_search_spaces.py diff --git a/tests_old/test_warm_starts/__init__.py b/tests_old/test_warm_starts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_warm_starts/test_memory_warm_start.py b/tests_old/test_warm_starts/test_memory_warm_start.py similarity index 100% rename from tests/test_warm_starts/test_memory_warm_start.py rename to tests_old/test_warm_starts/test_memory_warm_start.py diff --git a/tests/test_warm_starts/test_warm_start.py b/tests_old/test_warm_starts/test_warm_start.py similarity index 100% rename from tests/test_warm_starts/test_warm_start.py rename to tests_old/test_warm_starts/test_warm_start.py diff --git a/tests/test_warm_starts/test_warm_start_smbo.py b/tests_old/test_warm_starts/test_warm_start_smbo.py similarity index 100% rename from tests/test_warm_starts/test_warm_start_smbo.py rename to tests_old/test_warm_starts/test_warm_start_smbo.py From 19719e21b6276218b87871550df775b5e6083348 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 16 Mar 2025 18:31:20 +0100 Subject: [PATCH 27/62] implement sklearn-approved experiment class --- src/hyperactive/base/_experiment.py | 12 +++++++++--- src/hyperactive/optimizers/objective_function.py | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py index 88375f56..dbe98f0d 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/base/_experiment.py @@ -4,9 +4,12 @@ from skbase.base import BaseObject + + + class BaseExperiment(BaseObject): """Base class for experiment.""" - + def __init__( self, callbacks: Dict[str, callable] = None, @@ -14,8 +17,11 @@ def __init__( ): super().__init__() - self.callbacks = callbacks or {} - self.catch = catch or {} + self.callbacks = callbacks + self.catch = catch + + self._callbacks = callbacks or {} + self._catch = catch or {} def __call__(self, **kwargs): """Score parameters, with kwargs call.""" diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/objective_function.py index a3ecfced..03f73b06 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/objective_function.py @@ -20,8 +20,8 @@ def __init__(self, experiment): super().__init__() self.objective_function = experiment.objective_function - self.callbacks = experiment.callbacks - self.catch = experiment.catch + self.callbacks = experiment._callbacks + self.catch = experiment._catch self.nth_iter = 0 From acbfa255a2ab09f2206f769893b236741b041c90 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 16 Mar 2025 18:31:43 +0100 Subject: [PATCH 28/62] small fix --- src/hyperactive/optimizers/backend_stuff/distribution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperactive/optimizers/backend_stuff/distribution.py b/src/hyperactive/optimizers/backend_stuff/distribution.py index 4770aaad..b9932e2b 100644 --- a/src/hyperactive/optimizers/backend_stuff/distribution.py +++ b/src/hyperactive/optimizers/backend_stuff/distribution.py @@ -14,7 +14,7 @@ def single_process(process_func, process_infos): - return [process_func(*info) for info in process_infos] + return [process_func(info) for info in process_infos] def multiprocessing_wrapper(process_func, process_infos, n_processes): From 56b625999e0cbaa29b6211068b375e8d9278934a Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 11:45:06 +0200 Subject: [PATCH 29/62] add test dir for api; add callback test --- tests/test_api/__init__.py | 0 tests/test_api/test_callbacks.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/test_api/__init__.py create mode 100644 tests/test_api/test_callbacks.py diff --git a/tests/test_api/__init__.py b/tests/test_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_api/test_callbacks.py b/tests/test_api/test_callbacks.py new file mode 100644 index 00000000..78d08b68 --- /dev/null +++ b/tests/test_api/test_callbacks.py @@ -0,0 +1,38 @@ +import copy +import pytest +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.base import BaseExperiment +from hyperactive.search_config import SearchConfig + + +def test_callback_0(): + def callback_1(access): + access.stuff1 = 1 + + def callback_2(access): + access.stuff2 = 2 + + class Experiment(BaseExperiment): + def objective_function(self, access): + assert access.stuff1 == 1 + assert access.stuff2 == 2 + + return 0 + + search_config = SearchConfig( + x0=list(np.arange(2, 15, 1)), + x1=list(np.arange(2, 25, 2)), + ) + + objective_function = Experiment(callbacks={"before": [callback_1, callback_2]}) + + hyper = HillClimbingOptimizer() + hyper.add_search( + objective_function, + search_config, + n_iter=20, + ) + hyper.run() From 9b5c0a85f2daecd68d5f907295107a3daef36fed Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 13:18:40 +0200 Subject: [PATCH 30/62] move callbacks feature to separate decorator function --- src/hyperactive/base/_optimizer.py | 0 src/hyperactive/{base => experiment}/__init__.py | 3 ++- src/hyperactive/{base => experiment}/_experiment.py | 8 +------- src/hyperactive/experiment/_utility.py | 13 +++++++++++++ src/hyperactive/optimizers/objective_function.py | 7 ------- 5 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 src/hyperactive/base/_optimizer.py rename src/hyperactive/{base => experiment}/__init__.py (53%) rename src/hyperactive/{base => experiment}/_experiment.py (93%) create mode 100644 src/hyperactive/experiment/_utility.py diff --git a/src/hyperactive/base/_optimizer.py b/src/hyperactive/base/_optimizer.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/hyperactive/base/__init__.py b/src/hyperactive/experiment/__init__.py similarity index 53% rename from src/hyperactive/base/__init__.py rename to src/hyperactive/experiment/__init__.py index a237c3fe..1e3abf38 100644 --- a/src/hyperactive/base/__init__.py +++ b/src/hyperactive/experiment/__init__.py @@ -1,5 +1,6 @@ """Base classes for optimizers and experiments.""" from ._experiment import BaseExperiment +from ._utility import add_callbacks -__all__ = ["BaseExperiment"] +__all__ = ["BaseExperiment", "add_callbacks"] diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/experiment/_experiment.py similarity index 93% rename from src/hyperactive/base/_experiment.py rename to src/hyperactive/experiment/_experiment.py index dbe98f0d..fa2d394f 100644 --- a/src/hyperactive/base/_experiment.py +++ b/src/hyperactive/experiment/_experiment.py @@ -4,23 +4,17 @@ from skbase.base import BaseObject - - - class BaseExperiment(BaseObject): """Base class for experiment.""" - + def __init__( self, - callbacks: Dict[str, callable] = None, catch: Dict = None, ): super().__init__() - self.callbacks = callbacks self.catch = catch - self._callbacks = callbacks or {} self._catch = catch or {} def __call__(self, **kwargs): diff --git a/src/hyperactive/experiment/_utility.py b/src/hyperactive/experiment/_utility.py new file mode 100644 index 00000000..e921b7ad --- /dev/null +++ b/src/hyperactive/experiment/_utility.py @@ -0,0 +1,13 @@ +def add_callbacks(before=None, after=None): + def decorator(function): + def wrapper(self, param): + if before: + [before_callback(self, param) for before_callback in before] + result = function(self, param) + if after: + [after_callback(self, param) for after_callback in after] + return result + + return wrapper + + return decorator diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/objective_function.py index 03f73b06..6b54f4c0 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/objective_function.py @@ -20,15 +20,10 @@ def __init__(self, experiment): super().__init__() self.objective_function = experiment.objective_function - self.callbacks = experiment._callbacks self.catch = experiment._catch self.nth_iter = 0 - def run_callbacks(self, type_): - if self.callbacks and type_ in self.callbacks: - [callback(self) for callback in self.callbacks[type_]] - def __call__(self, search_space): # wrapper for GFOs def _model(para): @@ -37,9 +32,7 @@ def _model(para): self.para_dict = para try: - self.run_callbacks("before") results = self.objective_function(self) - self.run_callbacks("after") except tuple(self.catch.keys()) as e: results = self.catch[e.__class__] From dca41b6d1b62b6a4738e440431c7603b8334ca7a Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 13:19:41 +0200 Subject: [PATCH 31/62] add callback tests; adapt tests to new callbacks api --- tests/test_api/test_callbacks.py | 78 +++++++++++++++++++++++++++----- tests/test_setup/experiment.py | 4 +- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/tests/test_api/test_callbacks.py b/tests/test_api/test_callbacks.py index 78d08b68..51009029 100644 --- a/tests/test_api/test_callbacks.py +++ b/tests/test_api/test_callbacks.py @@ -4,35 +4,91 @@ import pandas as pd from hyperactive.optimizers import HillClimbingOptimizer -from hyperactive.base import BaseExperiment +from hyperactive.experiment import BaseExperiment, add_callbacks from hyperactive.search_config import SearchConfig -def test_callback_0(): - def callback_1(access): - access.stuff1 = 1 +search_config = SearchConfig( + x0=list(np.arange(-10, 10, 1)), +) + - def callback_2(access): - access.stuff2 = 2 +def test_callback_0(): class Experiment(BaseExperiment): + def callback_1(self, access): + access.stuff1 = 1 + + def callback_2(self, access): + access.stuff2 = 2 + + @add_callbacks(before=[callback_1, callback_2]) def objective_function(self, access): assert access.stuff1 == 1 assert access.stuff2 == 2 return 0 - search_config = SearchConfig( - x0=list(np.arange(2, 15, 1)), - x1=list(np.arange(2, 25, 2)), + objective_function = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + objective_function, + search_config, + n_iter=20, ) + hyper.run() + + +def test_callback_1(): + + class Experiment(BaseExperiment): + def callback_1(self, access): + access.stuff1 = 1 - objective_function = Experiment(callbacks={"before": [callback_1, callback_2]}) + def callback_2(self, access): + access.stuff1 = 2 + + @add_callbacks(before=[callback_1], after=[callback_2]) + def objective_function(self, access): + assert access.stuff1 == 1 + + return 0 + + objective_function = Experiment() hyper = HillClimbingOptimizer() hyper.add_search( objective_function, search_config, - n_iter=20, + n_iter=100, + ) + hyper.run() + + +def test_callback_2(): + + class Experiment(BaseExperiment): + + def callback_1(self, access): + self.test_var = 1 + + def setup(self, test_var): + self.test_var = test_var + + @add_callbacks(before=[callback_1]) + def objective_function(self, access): + assert self.test_var == 1 + + return 0 + + objective_function = Experiment() + objective_function.setup(5) + + hyper = HillClimbingOptimizer() + hyper.add_search( + objective_function, + search_config, + n_iter=100, ) hyper.run() diff --git a/tests/test_setup/experiment.py b/tests/test_setup/experiment.py index bea4deb7..5907e4ec 100644 --- a/tests/test_setup/experiment.py +++ b/tests/test_setup/experiment.py @@ -1,6 +1,6 @@ import numpy as np -from hyperactive.base import BaseExperiment +from hyperactive.experiment import BaseExperiment class SphereFunction(BaseExperiment): @@ -8,4 +8,4 @@ def setup(self, n_dim=2): self.n_dim = n_dim def objective_function(self, params): - return np.sum(params["x0"] ** 2 + params["x1"] ** 2) \ No newline at end of file + return np.sum(params["x0"] ** 2 + params["x1"] ** 2) From 09b01e4e216449abe68a99e2c083d034af36e3a9 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 13:45:29 +0200 Subject: [PATCH 32/62] rename variables --- tests/test_api/test_callbacks.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tests/test_api/test_callbacks.py b/tests/test_api/test_callbacks.py index 51009029..09a03eed 100644 --- a/tests/test_api/test_callbacks.py +++ b/tests/test_api/test_callbacks.py @@ -4,7 +4,7 @@ import pandas as pd from hyperactive.optimizers import HillClimbingOptimizer -from hyperactive.experiment import BaseExperiment, add_callbacks +from hyperactive.experiment import BaseExperiment, add_callback from hyperactive.search_config import SearchConfig @@ -14,7 +14,6 @@ def test_callback_0(): - class Experiment(BaseExperiment): def callback_1(self, access): access.stuff1 = 1 @@ -22,18 +21,18 @@ def callback_1(self, access): def callback_2(self, access): access.stuff2 = 2 - @add_callbacks(before=[callback_1, callback_2]) + @add_callback(before=[callback_1, callback_2]) def objective_function(self, access): assert access.stuff1 == 1 assert access.stuff2 == 2 return 0 - objective_function = Experiment() + experiment = Experiment() hyper = HillClimbingOptimizer() hyper.add_search( - objective_function, + experiment, search_config, n_iter=20, ) @@ -41,7 +40,6 @@ def objective_function(self, access): def test_callback_1(): - class Experiment(BaseExperiment): def callback_1(self, access): access.stuff1 = 1 @@ -49,17 +47,17 @@ def callback_1(self, access): def callback_2(self, access): access.stuff1 = 2 - @add_callbacks(before=[callback_1], after=[callback_2]) + @add_callback(before=[callback_1], after=[callback_2]) def objective_function(self, access): assert access.stuff1 == 1 return 0 - objective_function = Experiment() + experiment = Experiment() hyper = HillClimbingOptimizer() hyper.add_search( - objective_function, + experiment, search_config, n_iter=100, ) @@ -67,7 +65,6 @@ def objective_function(self, access): def test_callback_2(): - class Experiment(BaseExperiment): def callback_1(self, access): @@ -76,18 +73,18 @@ def callback_1(self, access): def setup(self, test_var): self.test_var = test_var - @add_callbacks(before=[callback_1]) + @add_callback(before=[callback_1]) def objective_function(self, access): assert self.test_var == 1 return 0 - objective_function = Experiment() - objective_function.setup(5) + experiment = Experiment() + experiment.setup(5) hyper = HillClimbingOptimizer() hyper.add_search( - objective_function, + experiment, search_config, n_iter=100, ) From a10d915a3cf3c87375905110ff2b4cb1fa83380d Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 13:45:42 +0200 Subject: [PATCH 33/62] small fix --- src/hyperactive/experiment/__init__.py | 4 ++-- src/hyperactive/experiment/_utility.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/hyperactive/experiment/__init__.py b/src/hyperactive/experiment/__init__.py index 1e3abf38..deb4ba6b 100644 --- a/src/hyperactive/experiment/__init__.py +++ b/src/hyperactive/experiment/__init__.py @@ -1,6 +1,6 @@ """Base classes for optimizers and experiments.""" from ._experiment import BaseExperiment -from ._utility import add_callbacks +from ._utility import add_callback, add_catch -__all__ = ["BaseExperiment", "add_callbacks"] +__all__ = ["BaseExperiment", "add_callback", "add_catch"] diff --git a/src/hyperactive/experiment/_utility.py b/src/hyperactive/experiment/_utility.py index e921b7ad..f4e38e59 100644 --- a/src/hyperactive/experiment/_utility.py +++ b/src/hyperactive/experiment/_utility.py @@ -1,4 +1,4 @@ -def add_callbacks(before=None, after=None): +def add_callback(before=None, after=None): def decorator(function): def wrapper(self, param): if before: @@ -11,3 +11,18 @@ def wrapper(self, param): return wrapper return decorator + + +def add_catch(catch): + def decorator(function): + def wrapper(self, param): + try: + result = function(self, param) + except tuple(catch.keys()) as e: + result = catch[e.__class__] + + return result + + return wrapper + + return decorator From 0fd35cd47188d23e8824347f479628f2b787591c Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 13:46:22 +0200 Subject: [PATCH 34/62] add tests for catch --- tests/test_api/test_catch.py | 137 +++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/test_api/test_catch.py diff --git a/tests/test_api/test_catch.py b/tests/test_api/test_catch.py new file mode 100644 index 00000000..29a05858 --- /dev/null +++ b/tests/test_api/test_catch.py @@ -0,0 +1,137 @@ +import copy +import pytest +import math +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment, add_catch +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x0=list(np.arange(-10, 10, 1)), +) + + +def test_catch_1(): + class Experiment(BaseExperiment): + @add_catch({TypeError: np.nan}) + def objective_function(self, access): + a = 1 + "str" + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + +def test_catch_2(): + class Experiment(BaseExperiment): + @add_catch({ValueError: np.nan}) + def objective_function(self, access): + math.sqrt(-10) + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + +def test_catch_3(): + class Experiment(BaseExperiment): + @add_catch({ZeroDivisionError: np.nan}) + def objective_function(self, access): + x = 1 / 0 + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + +def test_catch_all_0(): + class Experiment(BaseExperiment): + @add_catch( + { + TypeError: np.nan, + ValueError: np.nan, + ZeroDivisionError: np.nan, + } + ) + def objective_function(self, access): + a = 1 + "str" + math.sqrt(-10) + x = 1 / 0 + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + nan_ = hyper.search_data(experiment)["score"].values[0] + + assert math.isnan(nan_) + + +def test_catch_all_1(): + catch_return = (np.nan, {"error": True}) + + class Experiment(BaseExperiment): + @add_catch( + { + TypeError: catch_return, + ValueError: catch_return, + ZeroDivisionError: catch_return, + } + ) + def objective_function(self, access): + a = 1 + "str" + math.sqrt(-10) + x = 1 / 0 + + return 0, {"error": False} + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + nan_ = hyper.search_data(experiment)["score"].values[0] + error_ = hyper.search_data(experiment)["error"].values[0] + + assert math.isnan(nan_) + assert error_ == True From 44638c865597c3a816533ab2cd512d28c557d86d Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 13:48:46 +0200 Subject: [PATCH 35/62] add callback tests --- tests/test_api/test_callbacks.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_api/test_callbacks.py b/tests/test_api/test_callbacks.py index 09a03eed..f53d88b5 100644 --- a/tests/test_api/test_callbacks.py +++ b/tests/test_api/test_callbacks.py @@ -89,3 +89,33 @@ def objective_function(self, access): n_iter=100, ) hyper.run() + + +def test_callback_3(): + class Experiment(BaseExperiment): + + def callback_1(self, access): + access.pass_through["stuff1"] = 1 + + def setup(self, test_var): + self.test_var = test_var + + @add_callback(after=[callback_1]) + def objective_function(self, access): + if access.nth_iter == 0: + assert self.test_var == 0 + else: + assert self.test_var == 1 + + return 0 + + experiment = Experiment() + experiment.setup(0) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() From ce8e18859250a09a016d42eca1e0bd8c833674df Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 30 Mar 2025 16:30:19 +0200 Subject: [PATCH 36/62] fix nth_iter --- src/hyperactive/optimizers/objective_function.py | 4 ++-- tests/test_api/test_callbacks.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/objective_function.py index 6b54f4c0..25fed8b0 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/objective_function.py @@ -22,12 +22,12 @@ def __init__(self, experiment): self.objective_function = experiment.objective_function self.catch = experiment._catch - self.nth_iter = 0 + self.nth_iter = -1 def __call__(self, search_space): # wrapper for GFOs def _model(para): - # self.nth_iter = len(self.optimizer.pos_l) + self.nth_iter += 1 para = gfo2hyper(search_space, para) self.para_dict = para diff --git a/tests/test_api/test_callbacks.py b/tests/test_api/test_callbacks.py index f53d88b5..cdac7d73 100644 --- a/tests/test_api/test_callbacks.py +++ b/tests/test_api/test_callbacks.py @@ -12,6 +12,8 @@ x0=list(np.arange(-10, 10, 1)), ) +n_iter = 20 + def test_callback_0(): class Experiment(BaseExperiment): @@ -34,7 +36,7 @@ def objective_function(self, access): hyper.add_search( experiment, search_config, - n_iter=20, + n_iter=n_iter, ) hyper.run() @@ -59,7 +61,7 @@ def objective_function(self, access): hyper.add_search( experiment, search_config, - n_iter=100, + n_iter=n_iter, ) hyper.run() @@ -86,7 +88,7 @@ def objective_function(self, access): hyper.add_search( experiment, search_config, - n_iter=100, + n_iter=n_iter, ) hyper.run() @@ -95,7 +97,7 @@ def test_callback_3(): class Experiment(BaseExperiment): def callback_1(self, access): - access.pass_through["stuff1"] = 1 + self.test_var = 1 def setup(self, test_var): self.test_var = test_var @@ -116,6 +118,6 @@ def objective_function(self, access): hyper.add_search( experiment, search_config, - n_iter=100, + n_iter=n_iter, ) hyper.run() From ebec3bb496dea307e2029fa9e116e2f32ad4e9db Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Mon, 31 Mar 2025 18:34:12 +0200 Subject: [PATCH 37/62] add early_stop tests --- tests/test_api/test_early_stop.py | 329 ++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 tests/test_api/test_early_stop.py diff --git a/tests/test_api/test_early_stop.py b/tests/test_api/test_early_stop.py new file mode 100644 index 00000000..70caed29 --- /dev/null +++ b/tests/test_api/test_early_stop.py @@ -0,0 +1,329 @@ +import time +import pytest +import numpy as np + +from hyperactive.optimizers import RandomSearchOptimizer, HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x0"] * para["x0"] + return score + + +experiment = Experiment() +search_space = { + "x0": list(np.arange(0, 100000, 0.1)), +} +search_config = SearchConfig( + x0=list(np.arange(-10, 10, 1)), +) + + +def test_early_stop_0(): + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": 0.1, + "tol_rel": 0.1, + } + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + +def test_early_stop_1(): + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": None, + "tol_rel": 5, + } + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + +def test_early_stop_3(): + def objective_function(para): + score = -para["x0"] * para["x0"] + return score + + search_space = { + "x0": list(np.arange(0, 100, 0.1)), + } + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": n_iter_no_change, + } + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == (n_iter_no_change + 1) + + +def test_early_stop_4(): + def objective_function(para): + return para["x0"] + + search_space = { + "x0": list(np.arange(0, 100, 0.1)), + } + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": 0.1, + "tol_rel": None, + } + + start1 = {"x0": 0} + start2 = {"x0": 0.1} + start3 = {"x0": 0.2} + start4 = {"x0": 0.3} + start5 = {"x0": 0.4} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + start4, + start4, + start4, + start5, + start5, + start5, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == n_iter + + +def test_early_stop_5(): + def objective_function(para): + return para["x0"] + + search_space = { + "x0": list(np.arange(0, 100, 0.01)), + } + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": n_iter_no_change, + "tol_abs": 0.1, + "tol_rel": None, + } + + start1 = {"x0": 0} + start2 = {"x0": 0.09} + start3 = {"x0": 0.20} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == (n_iter_no_change + 1) + + +def test_early_stop_6(): + def objective_function(para): + return para["x0"] + + search_space = { + "x0": list(np.arange(0, 100, 0.01)), + } + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": None, + "tol_rel": 10, + } + + start1 = {"x0": 1} + start2 = {"x0": 1.1} + start3 = {"x0": 1.22} + start4 = {"x0": 1.35} + start5 = {"x0": 1.48} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + start4, + start4, + start4, + start5, + start5, + start5, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == n_iter + + +def test_early_stop_7(): + def objective_function(para): + return para["x0"] + + search_space = { + "x0": list(np.arange(0, 100, 0.01)), + } + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": n_iter_no_change, + "tol_abs": None, + "tol_rel": 10, + } + + start1 = {"x0": 1} + start2 = {"x0": 1.09} + start3 = {"x0": 1.20} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == (n_iter_no_change + 1) From 56f13358dc2ab4f4cdfa77e7ff69ef9769d61627 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Mon, 31 Mar 2025 18:37:30 +0200 Subject: [PATCH 38/62] fix search_config --- tests/test_api/test_early_stop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api/test_early_stop.py b/tests/test_api/test_early_stop.py index 70caed29..a15eacbc 100644 --- a/tests/test_api/test_early_stop.py +++ b/tests/test_api/test_early_stop.py @@ -18,7 +18,7 @@ def objective_function(self, para): "x0": list(np.arange(0, 100000, 0.1)), } search_config = SearchConfig( - x0=list(np.arange(-10, 10, 1)), + x0=list(np.arange(0, 100000, 0.1)), ) From 8a306bd101aee72d34721ff5a9bc4fe0e855823d Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Tue, 1 Apr 2025 16:31:07 +0200 Subject: [PATCH 39/62] add test_initializers.py --- tests/test_api/test_initializers.py | 128 ++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/test_api/test_initializers.py diff --git a/tests/test_api/test_initializers.py b/tests/test_api/test_initializers.py new file mode 100644 index 00000000..a798fe41 --- /dev/null +++ b/tests/test_api/test_initializers.py @@ -0,0 +1,128 @@ +import numpy as np +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + + +def test_initialize_warm_start_0(): + init = { + "x1": 0, + } + + initialize = {"warm_start": [init]} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) < 0.001 + + +def test_initialize_warm_start_1(): + search_space = { + "x1": list(np.arange(-10, 10, 1)), + } + init = { + "x1": -10, + } + + initialize = {"warm_start": [init]} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert hyper.best_para(experiment) == init + + +def test_initialize_vertices(): + initialize = {"vertices": 2} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=2, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) - 10000 < 0.001 + + +def test_initialize_grid_0(): + search_space = { + "x1": list(np.arange(-1, 2, 1)), + } + initialize = {"grid": 1} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) < 0.001 + + +def test_initialize_grid_1(): + search_space = { + "x1": list(np.arange(-2, 3, 1)), + } + + initialize = {"grid": 1} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) - 1 < 0.001 + + +def test_initialize_all_0(): + search_space = { + "x1": list(np.arange(-2, 3, 1)), + } + + initialize = {"grid": 100, "vertices": 100, "random": 100} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=300, + initialize=initialize, + ) + hyper.run() From 5ffca250161e4ad85a92b9f8194e55b4e435ddc2 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Tue, 1 Apr 2025 16:31:22 +0200 Subject: [PATCH 40/62] add test_random_state.py --- tests/test_api/test_random_state.py | 201 ++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 tests/test_api/test_random_state.py diff --git a/tests/test_api/test_random_state.py b/tests/test_api/test_random_state.py new file mode 100644 index 00000000..c155ab03 --- /dev/null +++ b/tests/test_api/test_random_state.py @@ -0,0 +1,201 @@ +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -(opt["x1"] * opt["x1"] + opt["x2"] * opt["x2"]) + return score + + +experiment = Experiment() + + +search_config = SearchConfig( + x1=list(np.arange(-1000, 1000, 0.1)), + x2=list(np.arange(-1000, 1000, 0.1)), +) + + +err = 0.001 + + +def test_random_state_n_jobs_0(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=5, + initialize={"random": 1}, + random_state=1, + n_jobs=n_jobs, + ) + hyper.run() + + results = hyper.search_data(experiment) + + no_dup = results.drop_duplicates(subset=list(search_space.keys())) + print("no_dup", no_dup) + print("results", results) + + print(int(len(results) / n_jobs)) + print(len(no_dup)) + + assert int(len(results) / n_jobs) != len(no_dup) + + +def test_random_state_n_jobs_1(): + n_jobs = 3 + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=5, + initialize={"random": 1}, + random_state=1, + n_jobs=n_jobs, + ) + hyper.run() + + results = hyper.search_data(experiment) + + no_dup = results.drop_duplicates(subset=list(search_space.keys())) + print("no_dup", no_dup) + print("results", results) + + assert int(len(results) / n_jobs) != len(no_dup) + + +def test_random_state_n_jobs_2(): + n_jobs = 4 + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=5, + initialize={"random": 1}, + random_state=1, + n_jobs=n_jobs, + ) + hyper.run() + + results = hyper.search_data(experiment) + + no_dup = results.drop_duplicates(subset=list(search_space.keys())) + print("no_dup", no_dup) + print("results", results) + + assert int(len(results) / n_jobs) != len(no_dup) + + +def test_random_state_0(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=1, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=1, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) < err + + +def test_random_state_1(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=10, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=10, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) < err + + +def test_random_state_2(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=1, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=10, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) > err + + +def test_no_random_state_0(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) > err From 30c124004938c02ef9023dad612877eef47d7ec3 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Tue, 1 Apr 2025 16:40:20 +0200 Subject: [PATCH 41/62] add test_search_spaces.py --- tests/test_api/test_search_spaces.py | 185 +++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 tests/test_api/test_search_spaces.py diff --git a/tests/test_api/test_search_spaces.py b/tests/test_api/test_search_spaces.py new file mode 100644 index 00000000..a8d1225e --- /dev/null +++ b/tests/test_api/test_search_spaces.py @@ -0,0 +1,185 @@ +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + + +def test_search_space_0(): + search_config = SearchConfig( + x1=list(np.arange(0, 3, 1)), + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["x1"] in search_config["x1"] + + +def test_search_space_1(): + search_config = SearchConfig( + x1=list(np.arange(0, 0.003, 0.001)), + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["x1"] in search_config["x1"] + + +def test_search_space_2(): + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + str1=["0", "1", "2"], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["str1"] in search_config["str1"] + + +def test_search_space_3(): + def func1(): + pass + + def func2(): + pass + + def func3(): + pass + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + func1=[func1, func2, func3], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["func1"] in search_config["func1"] + + +def test_search_space_4(): + class class1: + pass + + class class2: + pass + + class class3: + pass + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + class1=[class1, class2, class3], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["class1"] in search_config["class1"] + + +def test_search_space_5(): + class class1: + def __init__(self): + pass + + class class2: + def __init__(self): + pass + + class class3: + def __init__(self): + pass + + def class_f1(): + return class1 + + def class_f2(): + return class2 + + def class_f3(): + return class3 + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + class1=[class_f1, class_f2, class_f3], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["class1"] in search_config["class1"] + + +def test_search_space_6(): + + def list_f1(): + return [0, 1] + + def list_f2(): + return [1, 0] + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + list1=[list_f1, list_f2], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["list1"] in search_config["list1"] From 10aece002cda3c653d35109cee8fde2971cfa32b Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Tue, 1 Apr 2025 16:51:18 +0200 Subject: [PATCH 42/62] add test_results.py --- tests/test_api/test_results.py | 226 +++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 tests/test_api/test_results.py diff --git a/tests/test_api/test_results.py b/tests/test_api/test_results.py new file mode 100644 index 00000000..c90a419f --- /dev/null +++ b/tests/test_api/test_results.py @@ -0,0 +1,226 @@ +import pytest +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), +) + + +def test_attributes_results_0(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + + +def test_attributes_results_1(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert set(search_config.keys()) < set(hyper.search_data(experiment).columns) + + +def test_attributes_results_2(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert "x1" in list(hyper.search_data(experiment).columns) + + +def test_attributes_results_3(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert "score" in list(hyper.search_data(experiment).columns) + + +def test_attributes_results_4(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize={"warm_start": [{"x1": 0}]}, + ) + hyper.run() + + assert 0 in list(hyper.search_data(experiment)["x1"].values) + + +def test_attributes_results_5(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize={"warm_start": [{"x1": 10}]}, + ) + hyper.run() + + print( + "\n x1_results \n", + list(hyper.search_data(experiment)["x1"].values), + ) + + assert 10 in list(hyper.search_data(experiment)["x1"].values) + + +def test_attributes_results_6(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + initialize={"random": 1}, + memory=False, + ) + hyper.run() + + x1_results = list(hyper.search_data(experiment)["x1"].values) + + print("\n x1_results \n", x1_results) + + assert len(set(x1_results)) < len(x1_results) + + +def test_attributes_results_7(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + with pytest.raises(Exception) as e_info: + search_data["eval_times"] + + +def test_attributes_results_8(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + with pytest.raises(Exception) as e_info: + search_data["iter_times"] + + +def test_attributes_results_9(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + ) + hyper.run() + + search_data = hyper.search_data(experiment, times=True) + search_data["iter_times"] + search_data["eval_times"] + + +""" +def test_attributes_results_7(): + def objective_function(para): + score = -para["x1"] * para["x1"] + return score + + search_space = { + "x1": np.arange(0, 10, 1), + } + + opt = RandomSearchOptimizer(search_space) + opt.search( + experiment, n_iter=20, initialize={"random": 1}, memory=True + ) + + x1_results = list(opt.results["x1"].values) + + print("\n x1_results \n", x1_results) + + assert len(set(x1_results)) == len(x1_results) + + +def test_attributes_results_8(): + def objective_function(para): + score = -para["x1"] * para["x1"] + return score + + search_space = { + "x1": np.arange(-10, 11, 1), + } + + results = pd.DataFrame(np.arange(-10, 10, 1), columns=["x1"]) + results["score"] = 0 + + opt = RandomSearchOptimizer(search_space) + opt.search( + experiment, + n_iter=100, + initialize={}, + memory=True, + memory_warm_start=results, + ) + + print("\n opt.results \n", opt.results) + + x1_results = list(opt.results["x1"].values) + + assert 10 == x1_results[0] +""" From 429d72c259b2993f6652a0a45d6ec7718df962e4 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Tue, 1 Apr 2025 17:00:38 +0200 Subject: [PATCH 43/62] add test_results_methods.py --- tests/test_api/test_results_methods.py | 245 +++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 tests/test_api/test_results_methods.py diff --git a/tests/test_api/test_results_methods.py b/tests/test_api/test_results_methods.py new file mode 100644 index 00000000..90f52127 --- /dev/null +++ b/tests/test_api/test_results_methods.py @@ -0,0 +1,245 @@ +import pytest +import numbers +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +class Experiment1(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() +experiment1 = Experiment1() + + +search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), +) + + +def test_attributes_best_score_objective_function_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) + + +def test_attributes_best_score_objective_function_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) + + +""" +def test_attributes_best_score_search_id_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) + + +def test_attributes_best_score_search_id_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + search_id="2", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) +""" + + +def test_attributes_best_para_objective_function_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para(experiment), dict) + + +def test_attributes_best_para_objective_function_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para(experiment), dict) + + +""" +def test_attributes_best_para_search_id_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para("1"), dict) + + +def test_attributes_best_para_search_id_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + search_id="2", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para("1"), dict) +""" + + +def test_attributes_results_objective_function_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + + +def test_attributes_results_objective_function_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + + +""" +def test_attributes_results_search_id_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data("1"), pd.DataFrame) + + +def test_attributes_results_search_id_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + search_id="2", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data("1"), pd.DataFrame) +""" + + +def test_attributes_result_errors_0(): + with pytest.raises(ValueError): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15) + hyper.run() + + hyper.best_para(experiment1) + + +def test_attributes_result_errors_1(): + with pytest.raises(ValueError): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15) + hyper.run() + + hyper.best_score(experiment1) + + +def test_attributes_result_errors_2(): + with pytest.raises(ValueError): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15) + hyper.run() + + hyper.search_data(experiment1) From bba2abf0810a2d079dcad6bd3cb1f519d5ee0821 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 5 Apr 2025 10:15:32 +0200 Subject: [PATCH 44/62] add test_max_score.py --- tests/test_api/test_max_score.py | 77 ++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_api/test_max_score.py diff --git a/tests/test_api/test_max_score.py b/tests/test_api/test_max_score.py new file mode 100644 index 00000000..da1dc780 --- /dev/null +++ b/tests/test_api/test_max_score.py @@ -0,0 +1,77 @@ +import time +import numpy as np +from sklearn.datasets import load_breast_cancer +from sklearn.model_selection import cross_val_score +from sklearn.tree import DecisionTreeClassifier + +from hyperactive.optimizers import HillClimbingOptimizer, RandomSearchOptimizer +from hyperactive.experiment import BaseExperiment, add_catch +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x1=list(np.arange(0, 100, 0.1)), +) + + +def test_max_score_0(): + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + experiment = Experiment() + + max_score = -9999 + + hyper = HillClimbingOptimizer( + epsilon=0.01, + rand_rest_p=0, + ) + hyper.add_search( + experiment, + search_config, + n_iter=100000, + initialize={"warm_start": [{"x1": 99}]}, + max_score=max_score, + ) + hyper.run() + + print("\n Results head \n", hyper.search_data(experiment).head()) + print("\n Results tail \n", hyper.search_data(experiment).tail()) + + print("\nN iter:", len(hyper.search_data(experiment))) + + assert -100 > hyper.best_score(experiment) > max_score + + +def test_max_score_1(): + + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + time.sleep(0.01) + return score + + experiment = Experiment() + + max_score = -9999 + + c_time = time.perf_counter() + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100000, + initialize={"warm_start": [{"x1": 99}]}, + max_score=max_score, + ) + hyper.run() + diff_time = time.perf_counter() - c_time + + print("\n Results head \n", hyper.search_data(experiment).head()) + print("\n Results tail \n", hyper.search_data(experiment).tail()) + + print("\nN iter:", len(hyper.search_data(experiment))) + + assert diff_time < 1 From 3dc50a1fe74fda3730248f546d897b595889c2c9 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 5 Apr 2025 10:17:01 +0200 Subject: [PATCH 45/62] add test_max_time.py --- tests/test_api/test_max_time.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/test_api/test_max_time.py diff --git a/tests/test_api/test_max_time.py b/tests/test_api/test_max_time.py new file mode 100644 index 00000000..a9be4a42 --- /dev/null +++ b/tests/test_api/test_max_time.py @@ -0,0 +1,38 @@ +import time +import numpy as np +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment, add_catch +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(0, 100000, 1)), +) + + +def test_max_time_0(): + c_time1 = time.perf_counter() + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=1000000) + hyper.run(max_time=0.1) + diff_time1 = time.perf_counter() - c_time1 + + assert diff_time1 < 1 + + +def test_max_time_1(): + c_time1 = time.perf_counter() + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=1000000) + hyper.run(max_time=1) + diff_time1 = time.perf_counter() - c_time1 + + assert 0.3 < diff_time1 < 2 From 6e87b0b7acf8c87d3a276b929b509a0d35432a1b Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 5 Apr 2025 10:32:56 +0200 Subject: [PATCH 46/62] cleanup imports --- tests/test_api/test_max_score.py | 2 +- tests/test_api/test_max_time.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_api/test_max_score.py b/tests/test_api/test_max_score.py index da1dc780..0d46696a 100644 --- a/tests/test_api/test_max_score.py +++ b/tests/test_api/test_max_score.py @@ -5,7 +5,7 @@ from sklearn.tree import DecisionTreeClassifier from hyperactive.optimizers import HillClimbingOptimizer, RandomSearchOptimizer -from hyperactive.experiment import BaseExperiment, add_catch +from hyperactive.experiment import BaseExperiment from hyperactive.search_config import SearchConfig diff --git a/tests/test_api/test_max_time.py b/tests/test_api/test_max_time.py index a9be4a42..a1d4f00f 100644 --- a/tests/test_api/test_max_time.py +++ b/tests/test_api/test_max_time.py @@ -1,7 +1,7 @@ import time import numpy as np from hyperactive.optimizers import HillClimbingOptimizer -from hyperactive.experiment import BaseExperiment, add_catch +from hyperactive.experiment import BaseExperiment from hyperactive.search_config import SearchConfig From a19d64caf9cbc013636c69cf1bfd04551eda8a32 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 5 Apr 2025 10:33:07 +0200 Subject: [PATCH 47/62] add test_constr_opt.py --- tests/test_api/test_constr_opt.py | 152 ++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/test_api/test_constr_opt.py diff --git a/tests/test_api/test_constr_opt.py b/tests/test_api/test_constr_opt.py new file mode 100644 index 00000000..7da082bf --- /dev/null +++ b/tests/test_api/test_constr_opt.py @@ -0,0 +1,152 @@ +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +def test_constr_opt_0(): + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + experiment = Experiment() + + search_config = SearchConfig( + x1=list(np.arange(-15, 15, 1)), + ) + + def constraint_1(para): + print(" para", para) + + return para["x1"] > -5 + + constraints_list = [constraint_1] + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=50, + constraints=constraints_list, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + x0_values = search_data["x1"].values + + print("\n search_data \n", search_data, "\n") + + assert np.all(x0_values > -5) + + +def test_constr_opt_1(): + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -(para["x1"] * para["x1"] + para["x2"] * para["x2"]) + return score + + experiment = Experiment() + + search_config = SearchConfig( + x1=list(np.arange(-10, 10, 1)), + x2=list(np.arange(-10, 10, 1)), + ) + + def constraint_1(para): + return para["x1"] > -5 + + constraints_list = [constraint_1] + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=50, + constraints=constraints_list, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + x0_values = search_data["x1"].values + + print("\n search_data \n", search_data, "\n") + + assert np.all(x0_values > -5) + + +def test_constr_opt_2(): + n_iter = 50 + + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + experiment = Experiment() + + search_config = SearchConfig( + x1=list(np.arange(-10, 10, 0.1)), + ) + + def constraint_1(para): + return para["x1"] > -5 + + def constraint_2(para): + return para["x1"] < 5 + + constraints_list = [constraint_1, constraint_2] + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=50, + constraints=constraints_list, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + x0_values = search_data["x1"].values + + print("\n search_data \n", search_data, "\n") + + assert np.all(x0_values > -5) + assert np.all(x0_values < 5) + + n_new_positions = 0 + n_new_scores = 0 + + n_current_positions = 0 + n_current_scores = 0 + + n_best_positions = 0 + n_best_scores = 0 + + for hyper_optimizer in hyper.opt_pros.values(): + optimizer = hyper_optimizer.gfo_optimizer + + n_new_positions = n_new_positions + len(optimizer.pos_new_list) + n_new_scores = n_new_scores + len(optimizer.score_new_list) + + n_current_positions = n_current_positions + len(optimizer.pos_current_list) + n_current_scores = n_current_scores + len(optimizer.score_current_list) + + n_best_positions = n_best_positions + len(optimizer.pos_best_list) + n_best_scores = n_best_scores + len(optimizer.score_best_list) + + print("\n optimizer", optimizer) + print(" n_new_positions", optimizer.pos_new_list) + print(" n_new_scores", optimizer.score_new_list) + + assert n_new_positions == n_iter + assert n_new_scores == n_iter + + assert n_current_positions == n_current_scores + assert n_current_positions <= n_new_positions + + assert n_best_positions == n_best_scores + assert n_best_positions <= n_new_positions + + assert n_new_positions == n_new_scores From 9a567bf8fbcd19d5ddedc80f7f9e42b790b4e952 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 10:45:50 +0200 Subject: [PATCH 48/62] fix float error in early stop test --- tests/test_api/test_early_stop.py | 63 ++++++++++++++++++------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/tests/test_api/test_early_stop.py b/tests/test_api/test_early_stop.py index a15eacbc..fbc4c8a4 100644 --- a/tests/test_api/test_early_stop.py +++ b/tests/test_api/test_early_stop.py @@ -102,25 +102,28 @@ def objective_function(para): def test_early_stop_4(): - def objective_function(para): - return para["x0"] + class Experiment(BaseExperiment): + def objective_function(self, para): + return para["x0"] - search_space = { - "x0": list(np.arange(0, 100, 0.1)), - } + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) n_iter_no_change = 5 early_stopping = { "n_iter_no_change": 5, - "tol_abs": 0.1, + "tol_abs": 1, "tol_rel": None, } start1 = {"x0": 0} - start2 = {"x0": 0.1} - start3 = {"x0": 0.2} - start4 = {"x0": 0.3} - start5 = {"x0": 0.4} + start2 = {"x0": 1} + start3 = {"x0": 2} + start4 = {"x0": 3} + start5 = {"x0": 4} warm_start_l = [ start1, @@ -163,23 +166,26 @@ def objective_function(para): def test_early_stop_5(): - def objective_function(para): - return para["x0"] + class Experiment(BaseExperiment): + def objective_function(self, para): + return para["x0"] - search_space = { - "x0": list(np.arange(0, 100, 0.01)), - } + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) n_iter_no_change = 5 early_stopping = { "n_iter_no_change": n_iter_no_change, - "tol_abs": 0.1, + "tol_abs": 10, "tol_rel": None, } start1 = {"x0": 0} - start2 = {"x0": 0.09} - start3 = {"x0": 0.20} + start2 = {"x0": 9} + start3 = {"x0": 20} warm_start_l = [ start1, @@ -216,12 +222,15 @@ def objective_function(para): def test_early_stop_6(): - def objective_function(para): - return para["x0"] + class Experiment(BaseExperiment): + def objective_function(self, para): + return para["x0"] - search_space = { - "x0": list(np.arange(0, 100, 0.01)), - } + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) n_iter_no_change = 5 early_stopping = { @@ -280,9 +289,11 @@ def test_early_stop_7(): def objective_function(para): return para["x0"] - search_space = { - "x0": list(np.arange(0, 100, 0.01)), - } + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) n_iter_no_change = 5 early_stopping = { From a33a5eeda235b1c4188dbf26d5d51717bb8612a9 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 10:46:09 +0200 Subject: [PATCH 49/62] remove test that uses non-public api --- tests/test_api/test_constr_opt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_api/test_constr_opt.py b/tests/test_api/test_constr_opt.py index 7da082bf..29890933 100644 --- a/tests/test_api/test_constr_opt.py +++ b/tests/test_api/test_constr_opt.py @@ -76,6 +76,7 @@ def constraint_1(para): assert np.all(x0_values > -5) +""" def test_constr_opt_2(): n_iter = 50 @@ -150,3 +151,4 @@ def constraint_2(para): assert n_best_positions <= n_new_positions assert n_new_positions == n_new_scores +""" From 3a33ba03f6c5a28ebd88426839cafa3519adb40c Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 10:47:29 +0200 Subject: [PATCH 50/62] fix test --- tests/test_api/test_random_state.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_api/test_random_state.py b/tests/test_api/test_random_state.py index c155ab03..7b79258c 100644 --- a/tests/test_api/test_random_state.py +++ b/tests/test_api/test_random_state.py @@ -39,7 +39,7 @@ def test_random_state_n_jobs_0(): results = hyper.search_data(experiment) - no_dup = results.drop_duplicates(subset=list(search_space.keys())) + no_dup = results.drop_duplicates(subset=list(search_config.keys())) print("no_dup", no_dup) print("results", results) @@ -65,7 +65,7 @@ def test_random_state_n_jobs_1(): results = hyper.search_data(experiment) - no_dup = results.drop_duplicates(subset=list(search_space.keys())) + no_dup = results.drop_duplicates(subset=list(search_config.keys())) print("no_dup", no_dup) print("results", results) @@ -88,7 +88,7 @@ def test_random_state_n_jobs_2(): results = hyper.search_data(experiment) - no_dup = results.drop_duplicates(subset=list(search_space.keys())) + no_dup = results.drop_duplicates(subset=list(search_config.keys())) print("no_dup", no_dup) print("results", results) From 909b346d433193c1f421f851c101ac791fedc68a Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 10:54:29 +0200 Subject: [PATCH 51/62] pass objective-function instead of experiment --- src/hyperactive/optimizers/_optimizer_api.py | 30 +++++++++----------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index f7dcdb1b..a06337a3 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -40,8 +40,10 @@ def check_list(search_space): for key in search_space.keys(): search_dim = search_space[key] - error_msg = "Value in '{}' of search space dictionary must be of type list".format( - key + error_msg = ( + "Value in '{}' of search space dictionary must be of type list".format( + key + ) ) if not isinstance(search_dim, list): print("Warning", error_msg) @@ -93,9 +95,7 @@ def add_search( pass_through = pass_through or {} early_stopping = early_stopping or {} - search_id = self._default_search_id( - search_id, experiment.objective_function - ) + search_id = self._default_search_id(search_id, experiment.objective_function) s_space = SearchSpace(search_space) self.verbosity = verbosity @@ -135,12 +135,12 @@ def run( self.comp_opt = CompositeOptimizer(self) self.comp_opt.run(max_time, distribution, n_processes, self.verbosity) - def best_para(self, id_): + def best_para(self, experiment): """ Retrieve the best parameters for a specific ID from the results. Parameters: - - id_ (int): The ID of the parameters to retrieve. + - experiment (int): The experiment of the optimization run. Returns: - Union[Dict[str, Union[int, float]], None]: The best parameters for the specified ID if found, otherwise None. @@ -149,33 +149,31 @@ def best_para(self, id_): - ValueError: If the objective function name is not recognized. """ - return self.comp_opt.results_.best_para(id_) + return self.comp_opt.results_.best_para(experiment.objective_function) - def best_score(self, id_): + def best_score(self, experiment): """ Return the best score for a specific ID from the results. Parameters: - - id_ (int): The ID for which the best score is requested. + - experiment (int): The experiment of the optimization run. """ - return self.comp_opt.results_.best_score(id_) + return self.comp_opt.results_.best_score(experiment.objective_function) - def search_data(self, id_, times=False): + def search_data(self, experiment, times=False): """ Retrieve search data for a specific ID from the results. Optionally exclude evaluation and iteration times if 'times' is set to False. Parameters: - - id_ (int): The ID of the search data to retrieve. + - experiment (int): The experiment of the optimization run. - times (bool, optional): Whether to exclude evaluation and iteration times. Defaults to False. Returns: - pd.DataFrame: The search data for the specified ID. """ - search_data_ = self.comp_opt.results_.search_data( - id_.objective_function - ) + search_data_ = self.comp_opt.results_.search_data(experiment.objective_function) if times == False: search_data_.drop( From ad12386527328d8faa2b138c41f3c3fb29befc4e Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 11:56:03 +0200 Subject: [PATCH 52/62] add test_distribution.py --- tests/test_api/test_distribution.py | 104 ++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/test_api/test_distribution.py diff --git a/tests/test_api/test_distribution.py b/tests/test_api/test_distribution.py new file mode 100644 index 00000000..4e39228e --- /dev/null +++ b/tests/test_api/test_distribution.py @@ -0,0 +1,104 @@ +import numpy as np +from tqdm import tqdm + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + +n_iter = 15 + + +def test_n_jobs_0(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs + + +def test_n_jobs_1(): + n_jobs = 4 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=n_jobs) + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs + + +def test_n_jobs_2(): + n_jobs = 8 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs + + +def test_n_jobs_5(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs * 2 + + +def test_n_jobs_6(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs * 4 + + +def test_n_jobs_7(): + n_jobs = -1 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=n_jobs) + hyper.run() + + +def test_multiprocessing_0(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=2) + hyper.run(distribution="multiprocessing") + + +def test_multiprocessing_1(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=2) + hyper.run( + distribution={ + "multiprocessing": { + "initializer": tqdm.set_lock, + "initargs": (tqdm.get_lock(),), + } + }, + ) From 32a2c8ef67a15a3a9a414bcfd0c5fce1f39f7765 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 12:04:00 +0200 Subject: [PATCH 53/62] add tests for cmd output --- tests/test_api/test_empty_output/__init__.py | 0 .../test_api/test_empty_output/non_verbose.py | 20 +++++++++ .../test_empty_output/test_empty_output.py | 41 +++++++++++++++++++ tests/test_api/test_empty_output/verbose.py | 26 ++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 tests/test_api/test_empty_output/__init__.py create mode 100644 tests/test_api/test_empty_output/non_verbose.py create mode 100644 tests/test_api/test_empty_output/test_empty_output.py create mode 100644 tests/test_api/test_empty_output/verbose.py diff --git a/tests/test_api/test_empty_output/__init__.py b/tests/test_api/test_empty_output/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_api/test_empty_output/non_verbose.py b/tests/test_api/test_empty_output/non_verbose.py new file mode 100644 index 00000000..3eadd4c6 --- /dev/null +++ b/tests/test_api/test_empty_output/non_verbose.py @@ -0,0 +1,20 @@ +import numpy as np +from hyperactive import Hyperactive + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + + +hyper = Hyperactive() +hyper.add_search(experiment, search_config, n_iter=30, memory=True) +hyper.run(verbosity=False) diff --git a/tests/test_api/test_empty_output/test_empty_output.py b/tests/test_api/test_empty_output/test_empty_output.py new file mode 100644 index 00000000..dddef388 --- /dev/null +++ b/tests/test_api/test_empty_output/test_empty_output.py @@ -0,0 +1,41 @@ +import os, sys, subprocess + +here = os.path.dirname(os.path.abspath(__file__)) + +verbose_file = os.path.join(here, "verbose.py") +non_verbose_file = os.path.join(here, "non_verbose.py") + + +def _run_subprocess(script): + output = [] + process = subprocess.Popen( + [sys.executable, "-u", script], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # Line buffered + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + ) + # Read output line by line + while True: + line = process.stdout.readline() + if line: + output.append(line) + if not line and process.poll() is not None: + break + + return "".join(output), process.stderr.read() + + +def test_empty_output(): + stdout_verb, stderr_verb = _run_subprocess(verbose_file) + stdout_non_verb, stderr_non_verb = _run_subprocess(non_verbose_file) + + print("\n stdout_verb \n", stdout_verb, "\n") + print("\n stderr_verb \n", stderr_verb, "\n") + + print("\n stdout_non_verb \n", stdout_non_verb, "\n") + print("\n stderr_non_verb \n", stderr_non_verb, "\n") + + assert "Results:" in stdout_verb + assert not stdout_non_verb diff --git a/tests/test_api/test_empty_output/verbose.py b/tests/test_api/test_empty_output/verbose.py new file mode 100644 index 00000000..a7e16a73 --- /dev/null +++ b/tests/test_api/test_empty_output/verbose.py @@ -0,0 +1,26 @@ +import sys +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + + +hyper = HillClimbingOptimizer() +hyper.add_search(experiment, search_config, n_iter=30, memory=True) +hyper.run() + +sys.stdout.flush() From 9fd44827bf0026e605935c11e4d428f58d8e3744 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 12:21:23 +0200 Subject: [PATCH 54/62] fix verbosity --- src/hyperactive/composite_optimizer.py | 13 +++++----- src/hyperactive/optimizers/_optimizer_api.py | 6 ++--- .../optimizers/backend_stuff/distribution.py | 15 ++++------- src/hyperactive/optimizers/search.py | 25 +++++++------------ 4 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/hyperactive/composite_optimizer.py b/src/hyperactive/composite_optimizer.py index d934c119..4405d7c1 100644 --- a/src/hyperactive/composite_optimizer.py +++ b/src/hyperactive/composite_optimizer.py @@ -21,14 +21,15 @@ def run( n_processes: Union[str, int] = "auto", verbosity: list = ["progress_bar", "print_results", "print_times"], ): - self.verbosity = verbosity + if not verbosity: + verbosity = [] self.collected_searches = [] for optimizer in self.optimizers: self.collected_searches += optimizer.searches for nth_process, search in enumerate(self.collected_searches): - search.pass_args(max_time, nth_process) + search.pass_args(max_time, nth_process, verbosity) self.results_list = run_search( self.collected_searches, distribution, n_processes @@ -36,12 +37,12 @@ def run( self.results_ = Results(self.results_list, self.collected_searches) - self._print_info() + self._print_info(verbosity) - def _print_info(self): - print_res = PrintResults(self.collected_searches, self.verbosity) + def _print_info(self, verbosity): + print_res = PrintResults(self.collected_searches, verbosity) - if self.verbosity: + if verbosity: for _ in range(len(self.collected_searches)): print("") diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index a06337a3..7393dcff 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -56,7 +56,6 @@ def add_search( n_iter: int, search_id=None, n_jobs: int = 1, - verbosity: list = ["progress_bar", "print_results", "print_times"], initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, constraints: List[callable] = None, pass_through: Dict = None, @@ -97,7 +96,6 @@ def add_search( search_id = self._default_search_id(search_id, experiment.objective_function) s_space = SearchSpace(search_space) - self.verbosity = verbosity n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs @@ -115,7 +113,6 @@ def add_search( random_state=random_state, memory=memory, memory_warm_start=memory_warm_start, - verbosity=verbosity, ) self.searches.append(search) @@ -131,9 +128,10 @@ def run( max_time=None, distribution: str = "multiprocessing", n_processes: Union[str, int] = "auto", + verbosity: list = ["progress_bar", "print_results", "print_times"], ): self.comp_opt = CompositeOptimizer(self) - self.comp_opt.run(max_time, distribution, n_processes, self.verbosity) + self.comp_opt.run(max_time, distribution, n_processes, verbosity) def best_para(self, experiment): """ diff --git a/src/hyperactive/optimizers/backend_stuff/distribution.py b/src/hyperactive/optimizers/backend_stuff/distribution.py index b9932e2b..44901c7b 100644 --- a/src/hyperactive/optimizers/backend_stuff/distribution.py +++ b/src/hyperactive/optimizers/backend_stuff/distribution.py @@ -22,26 +22,21 @@ def multiprocessing_wrapper(process_func, process_infos, n_processes): process_infos = tuple(process_infos) - with mp.Pool( - n_processes, initializer=initializer, initargs=initargs - ) as pool: + print("\n process_infos ", process_infos) + + with mp.Pool(n_processes, initializer=initializer, initargs=initargs) as pool: return pool.map(process_func, process_infos) def pathos_wrapper(process_func, search_processes_paras, n_processes): import pathos.multiprocessing as pmp - with pmp.Pool( - n_processes, initializer=initializer, initargs=initargs - ) as pool: + with pmp.Pool(n_processes, initializer=initializer, initargs=initargs) as pool: return pool.map(process_func, search_processes_paras) def joblib_wrapper(process_func, search_processes_paras, n_processes): from joblib import Parallel, delayed - jobs = [ - delayed(process_func)(*info_dict) - for info_dict in search_processes_paras - ] + jobs = [delayed(process_func)(*info_dict) for info_dict in search_processes_paras] return Parallel(n_jobs=n_processes)(jobs) diff --git a/src/hyperactive/optimizers/search.py b/src/hyperactive/optimizers/search.py index d51e5bf7..1281779f 100644 --- a/src/hyperactive/optimizers/search.py +++ b/src/hyperactive/optimizers/search.py @@ -30,7 +30,6 @@ def setup( random_state, memory, memory_warm_start, - verbosity, ): self.experiment = experiment self.s_space = s_space @@ -44,17 +43,16 @@ def setup( self.random_state = random_state self.memory = memory self.memory_warm_start = memory_warm_start - self.verbosity = verbosity - if "progress_bar" in self.verbosity: + def pass_args(self, max_time, nth_process, verbosity): + self.max_time = max_time + self.nth_process = nth_process + + if "progress_bar" in verbosity: self.verbosity = ["progress_bar"] else: self.verbosity = [] - def pass_args(self, max_time, nth_process): - self.max_time = max_time - self.nth_process = nth_process - def convert_results2hyper(self): self.eval_times = sum(self.gfo_optimizer.eval_times) self.iter_times = sum(self.gfo_optimizer.iter_times) @@ -86,15 +84,12 @@ def _setup_process(self): # conv warm start for smbo from values into positions if "warm_start_smbo" in self.opt_params: - self.opt_params["warm_start_smbo"] = ( - self.hg_conv.conv_memory_warm_start( - self.opt_params["warm_start_smbo"] - ) + self.opt_params["warm_start_smbo"] = self.hg_conv.conv_memory_warm_start( + self.opt_params["warm_start_smbo"] ) gfo_constraints = [ - Constraint(constraint, self.s_space) - for constraint in self.constraints + Constraint(constraint, self.s_space) for constraint in self.constraints ] self.gfo_optimizer = self.optimizer_class( @@ -116,9 +111,7 @@ def _search(self, p_bar): ) gfo_wrapper_model.pass_through = self.pass_through - memory_warm_start = self.hg_conv.conv_memory_warm_start( - self.memory_warm_start - ) + memory_warm_start = self.hg_conv.conv_memory_warm_start(self.memory_warm_start) gfo_objective_function = gfo_wrapper_model(self.s_space()) From b90f17619ce63f860b17ef7f37bbec50950d571c Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 12:21:38 +0200 Subject: [PATCH 55/62] separate verbosity tests --- tests/test_api/test_empty_output/non_verbose.py | 7 +++++-- .../test_empty_output/test_empty_output.py | 15 ++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_api/test_empty_output/non_verbose.py b/tests/test_api/test_empty_output/non_verbose.py index 3eadd4c6..8b313e50 100644 --- a/tests/test_api/test_empty_output/non_verbose.py +++ b/tests/test_api/test_empty_output/non_verbose.py @@ -1,5 +1,8 @@ import numpy as np -from hyperactive import Hyperactive + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig class Experiment(BaseExperiment): @@ -15,6 +18,6 @@ def objective_function(self, opt): ) -hyper = Hyperactive() +hyper = HillClimbingOptimizer() hyper.add_search(experiment, search_config, n_iter=30, memory=True) hyper.run(verbosity=False) diff --git a/tests/test_api/test_empty_output/test_empty_output.py b/tests/test_api/test_empty_output/test_empty_output.py index dddef388..cd3a3e33 100644 --- a/tests/test_api/test_empty_output/test_empty_output.py +++ b/tests/test_api/test_empty_output/test_empty_output.py @@ -28,14 +28,19 @@ def _run_subprocess(script): def test_empty_output(): - stdout_verb, stderr_verb = _run_subprocess(verbose_file) stdout_non_verb, stderr_non_verb = _run_subprocess(non_verbose_file) - print("\n stdout_verb \n", stdout_verb, "\n") - print("\n stderr_verb \n", stderr_verb, "\n") - print("\n stdout_non_verb \n", stdout_non_verb, "\n") print("\n stderr_non_verb \n", stderr_non_verb, "\n") - assert "Results:" in stdout_verb assert not stdout_non_verb + assert not stderr_non_verb + + +def test_output(): + stdout_verb, stderr_verb = _run_subprocess(verbose_file) + + print("\n stdout_verb \n", stdout_verb, "\n") + print("\n stderr_verb \n", stderr_verb, "\n") + + assert "Results:" in stdout_verb From 83f54312be93e7a98c17eab72a5bce65e162c83a Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sat, 12 Apr 2025 21:21:28 +0200 Subject: [PATCH 56/62] rename + delete files --- ...e_optimizer.py => _composite_optimizer.py} | 6 +- src/hyperactive/optimizers/__init__.py | 2 +- .../{constraint.py => _constraint.py} | 0 .../{dictionary.py => _dictionary.py} | 0 ...adient_conv.py => _hyper_gradient_conv.py} | 0 ...ive_function.py => _objective_function.py} | 2 +- src/hyperactive/optimizers/_optimizer_api.py | 4 +- ...attributes.py => _optimizer_attributes.py} | 0 .../{optimizers.py => _optimizers.py} | 0 .../optimizers/{search.py => _search.py} | 8 +- .../{distribution.py => _distribution.py} | 44 +++ .../{print_results.py => _print_results.py} | 0 .../backend_stuff/{process.py => _process.py} | 0 .../backend_stuff/{results.py => _results.py} | 0 .../optimizers/backend_stuff/hyperactive.py | 259 ------------------ .../optimizers/backend_stuff/run_search.py | 55 ---- 16 files changed, 55 insertions(+), 325 deletions(-) rename src/hyperactive/{composite_optimizer.py => _composite_optimizer.py} (88%) rename src/hyperactive/optimizers/{constraint.py => _constraint.py} (100%) rename src/hyperactive/optimizers/{dictionary.py => _dictionary.py} (100%) rename src/hyperactive/optimizers/{hyper_gradient_conv.py => _hyper_gradient_conv.py} (100%) rename src/hyperactive/optimizers/{objective_function.py => _objective_function.py} (96%) rename src/hyperactive/optimizers/{optimizer_attributes.py => _optimizer_attributes.py} (100%) rename src/hyperactive/optimizers/{optimizers.py => _optimizers.py} (100%) rename src/hyperactive/optimizers/{search.py => _search.py} (96%) rename src/hyperactive/optimizers/backend_stuff/{distribution.py => _distribution.py} (52%) rename src/hyperactive/optimizers/backend_stuff/{print_results.py => _print_results.py} (100%) rename src/hyperactive/optimizers/backend_stuff/{process.py => _process.py} (100%) rename src/hyperactive/optimizers/backend_stuff/{results.py => _results.py} (100%) delete mode 100644 src/hyperactive/optimizers/backend_stuff/hyperactive.py delete mode 100644 src/hyperactive/optimizers/backend_stuff/run_search.py diff --git a/src/hyperactive/composite_optimizer.py b/src/hyperactive/_composite_optimizer.py similarity index 88% rename from src/hyperactive/composite_optimizer.py rename to src/hyperactive/_composite_optimizer.py index 4405d7c1..3eadb82a 100644 --- a/src/hyperactive/composite_optimizer.py +++ b/src/hyperactive/_composite_optimizer.py @@ -1,7 +1,7 @@ from typing import Union -from .optimizers.backend_stuff.run_search import run_search -from .optimizers.backend_stuff.results import Results -from .optimizers.backend_stuff.print_results import PrintResults +from .optimizers.backend_stuff._distribution import run_search +from .optimizers.backend_stuff._results import Results +from .optimizers.backend_stuff._print_results import PrintResults class CompositeOptimizer: diff --git a/src/hyperactive/optimizers/__init__.py b/src/hyperactive/optimizers/__init__.py index c476012e..cd77760f 100644 --- a/src/hyperactive/optimizers/__init__.py +++ b/src/hyperactive/optimizers/__init__.py @@ -3,7 +3,7 @@ # License: MIT License -from .optimizers import ( +from ._optimizers import ( HillClimbingOptimizer, StochasticHillClimbingOptimizer, RepulsingHillClimbingOptimizer, diff --git a/src/hyperactive/optimizers/constraint.py b/src/hyperactive/optimizers/_constraint.py similarity index 100% rename from src/hyperactive/optimizers/constraint.py rename to src/hyperactive/optimizers/_constraint.py diff --git a/src/hyperactive/optimizers/dictionary.py b/src/hyperactive/optimizers/_dictionary.py similarity index 100% rename from src/hyperactive/optimizers/dictionary.py rename to src/hyperactive/optimizers/_dictionary.py diff --git a/src/hyperactive/optimizers/hyper_gradient_conv.py b/src/hyperactive/optimizers/_hyper_gradient_conv.py similarity index 100% rename from src/hyperactive/optimizers/hyper_gradient_conv.py rename to src/hyperactive/optimizers/_hyper_gradient_conv.py diff --git a/src/hyperactive/optimizers/objective_function.py b/src/hyperactive/optimizers/_objective_function.py similarity index 96% rename from src/hyperactive/optimizers/objective_function.py rename to src/hyperactive/optimizers/_objective_function.py index 25fed8b0..2fa67056 100644 --- a/src/hyperactive/optimizers/objective_function.py +++ b/src/hyperactive/optimizers/_objective_function.py @@ -3,7 +3,7 @@ # License: MIT License -from .dictionary import DictClass +from ._dictionary import DictClass def gfo2hyper(search_space, para): diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimizers/_optimizer_api.py index 7393dcff..e0b99769 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimizers/_optimizer_api.py @@ -5,10 +5,10 @@ import pandas as pd from .backend_stuff.search_space import SearchSpace -from .search import Search +from ._search import Search -from ..composite_optimizer import CompositeOptimizer +from .._composite_optimizer import CompositeOptimizer from skbase.base import BaseObject diff --git a/src/hyperactive/optimizers/optimizer_attributes.py b/src/hyperactive/optimizers/_optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimizers/optimizer_attributes.py rename to src/hyperactive/optimizers/_optimizer_attributes.py diff --git a/src/hyperactive/optimizers/optimizers.py b/src/hyperactive/optimizers/_optimizers.py similarity index 100% rename from src/hyperactive/optimizers/optimizers.py rename to src/hyperactive/optimizers/_optimizers.py diff --git a/src/hyperactive/optimizers/search.py b/src/hyperactive/optimizers/_search.py similarity index 96% rename from src/hyperactive/optimizers/search.py rename to src/hyperactive/optimizers/_search.py index 1281779f..cb9d8d3a 100644 --- a/src/hyperactive/optimizers/search.py +++ b/src/hyperactive/optimizers/_search.py @@ -2,10 +2,10 @@ # Email: simon.blanke@yahoo.com # License: MIT License -from .objective_function import ObjectiveFunction -from .hyper_gradient_conv import HyperGradientConv -from .optimizer_attributes import OptimizerAttributes -from .constraint import Constraint +from ._objective_function import ObjectiveFunction +from ._hyper_gradient_conv import HyperGradientConv +from ._optimizer_attributes import OptimizerAttributes +from ._constraint import Constraint class Search(OptimizerAttributes): diff --git a/src/hyperactive/optimizers/backend_stuff/distribution.py b/src/hyperactive/optimizers/backend_stuff/_distribution.py similarity index 52% rename from src/hyperactive/optimizers/backend_stuff/distribution.py rename to src/hyperactive/optimizers/backend_stuff/_distribution.py index 44901c7b..6b060f67 100644 --- a/src/hyperactive/optimizers/backend_stuff/distribution.py +++ b/src/hyperactive/optimizers/backend_stuff/_distribution.py @@ -5,6 +5,9 @@ from sys import platform from tqdm import tqdm +from ._process import _process_ + + if platform.startswith("linux"): initializer = tqdm.set_lock initargs = (tqdm.get_lock(),) @@ -13,6 +16,10 @@ initargs = () +def proxy(args): + return _process_(*args) + + def single_process(process_func, process_infos): return [process_func(info) for info in process_infos] @@ -40,3 +47,40 @@ def joblib_wrapper(process_func, search_processes_paras, n_processes): jobs = [delayed(process_func)(*info_dict) for info_dict in search_processes_paras] return Parallel(n_jobs=n_processes)(jobs) + + +dist_dict = { + "joblib": (joblib_wrapper, _process_), + "multiprocessing": (multiprocessing_wrapper, proxy), + "pathos": (pathos_wrapper, proxy), +} + + +def _get_distribution(distribution): + if hasattr(distribution, "__call__"): + return (distribution, _process_), {} + + elif isinstance(distribution, dict): + dist_key = list(distribution.keys())[0] + dist_paras = list(distribution.values())[0] + + return dist_dict[dist_key], dist_paras + + elif isinstance(distribution, str): + return dist_dict[distribution], {} + + +def run_search(searches, distribution, n_processes): + if n_processes == "auto": + n_processes = len(searches) + + searches_tuple = [(search,) for search in searches] + + if n_processes == 1: + results_list = single_process(_process_, searches) + else: + (distribution, process_func), dist_paras = _get_distribution(distribution) + + results_list = distribution(process_func, searches_tuple, n_processes) + + return results_list diff --git a/src/hyperactive/optimizers/backend_stuff/print_results.py b/src/hyperactive/optimizers/backend_stuff/_print_results.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/print_results.py rename to src/hyperactive/optimizers/backend_stuff/_print_results.py diff --git a/src/hyperactive/optimizers/backend_stuff/process.py b/src/hyperactive/optimizers/backend_stuff/_process.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/process.py rename to src/hyperactive/optimizers/backend_stuff/_process.py diff --git a/src/hyperactive/optimizers/backend_stuff/results.py b/src/hyperactive/optimizers/backend_stuff/_results.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/results.py rename to src/hyperactive/optimizers/backend_stuff/_results.py diff --git a/src/hyperactive/optimizers/backend_stuff/hyperactive.py b/src/hyperactive/optimizers/backend_stuff/hyperactive.py deleted file mode 100644 index 18face95..00000000 --- a/src/hyperactive/optimizers/backend_stuff/hyperactive.py +++ /dev/null @@ -1,259 +0,0 @@ -# Author: Simon Blanke -# Email: simon.blanke@yahoo.com -# License: MIT License - - -import copy -import multiprocessing as mp -import pandas as pd - -from typing import Union, List, Dict, Type - -from .optimizers import RandomSearchOptimizer -from .run_search import run_search - -from .results import Results -from .print_results import PrintResults -from .search_space import SearchSpace - - -class Hyperactive: - """ - Initialize the Hyperactive class to manage optimization processes. - - Parameters: - - verbosity: List of verbosity levels (default: ["progress_bar", "print_results", "print_times"]) - - distribution: String indicating the distribution method (default: "multiprocessing") - - n_processes: Number of processes to run in parallel or "auto" to determine automatically (default: "auto") - - Methods: - - add_search: Add a new optimization search process with specified parameters - - run: Execute the optimization searches - - best_para: Get the best parameters for a specific search - - best_score: Get the best score for a specific search - - search_data: Get the search data for a specific search - """ - - def __init__( - self, - verbosity: list = ["progress_bar", "print_results", "print_times"], - distribution: str = "multiprocessing", - n_processes: Union[str, int] = "auto", - ): - super().__init__() - if verbosity is False: - verbosity = [] - - self.verbosity = verbosity - self.distribution = distribution - self.n_processes = n_processes - - self.opt_pros = {} - - def _create_shared_memory(self): - _bundle_opt_processes = {} - - for opt_pros in self.opt_pros.values(): - if opt_pros.memory != "share": - continue - name = opt_pros.objective_function.__name__ - - _bundle_opt_processes.setdefault(name, []).append(opt_pros) - - for opt_pros_l in _bundle_opt_processes.values(): - # Check if the lengths of the search spaces of all optimizers in the list are the same. - if ( - len(set(len(opt_pros.s_space()) for opt_pros in opt_pros_l)) - == 1 - ): - manager = mp.Manager() # get new manager.dict - shared_memory = manager.dict() - for opt_pros in opt_pros_l: - opt_pros.memory = shared_memory - else: - for opt_pros in opt_pros_l: - opt_pros.memory = opt_pros_l[ - 0 - ].memory # get same manager.dict - - @staticmethod - def _default_opt(optimizer): - if isinstance(optimizer, str): - if optimizer == "default": - optimizer = RandomSearchOptimizer() - return copy.deepcopy(optimizer) - - @staticmethod - def _default_search_id(search_id, objective_function): - if not search_id: - search_id = objective_function.__name__ - return search_id - - @staticmethod - def check_list(search_space): - for key in search_space.keys(): - search_dim = search_space[key] - - error_msg = "Value in '{}' of search space dictionary must be of type list".format( - key - ) - if not isinstance(search_dim, list): - print("Warning", error_msg) - # raise ValueError(error_msg) - - def add_search( - self, - objective_function: callable, - search_space: Dict[str, list], - n_iter: int, - search_id=None, - optimizer: Union[str, Type[RandomSearchOptimizer]] = "default", - n_jobs: int = 1, - initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, - constraints: List[callable] = None, - pass_through: Dict = None, - callbacks: Dict[str, callable] = None, - catch: Dict = None, - max_score: float = None, - early_stopping: Dict = None, - random_state: int = None, - memory: Union[str, bool] = "share", - memory_warm_start: pd.DataFrame = None, - ): - """ - Add a new optimization search process with specified parameters. - - Parameters: - - objective_function: The objective function to optimize. - - search_space: Dictionary defining the search space for optimization. - - n_iter: Number of iterations for the optimization process. - - search_id: Identifier for the search process (default: None). - - optimizer: The optimizer to use for the search process (default: "default"). - - n_jobs: Number of parallel jobs to run (default: 1). - - initialize: Dictionary specifying initialization parameters (default: {"grid": 4, "random": 2, "vertices": 4}). - - constraints: List of constraint functions (default: None). - - pass_through: Dictionary of additional parameters to pass through (default: None). - - callbacks: Dictionary of callback functions (default: None). - - catch: Dictionary of exceptions to catch during optimization (default: None). - - max_score: Maximum score to achieve (default: None). - - early_stopping: Dictionary specifying early stopping criteria (default: None). - - random_state: Seed for random number generation (default: None). - - memory: Option to share memory between processes (default: "share"). - - memory_warm_start: DataFrame containing warm start memory (default: None). - """ - - self.check_list(search_space) - - constraints = constraints or [] - pass_through = pass_through or {} - callbacks = callbacks or {} - catch = catch or {} - early_stopping = early_stopping or {} - - optimizer = self._default_opt(optimizer) - search_id = self._default_search_id(search_id, objective_function) - s_space = SearchSpace(search_space) - - optimizer.setup_search( - objective_function=objective_function, - s_space=s_space, - n_iter=n_iter, - initialize=initialize, - constraints=constraints, - pass_through=pass_through, - callbacks=callbacks, - catch=catch, - max_score=max_score, - early_stopping=early_stopping, - random_state=random_state, - memory=memory, - memory_warm_start=memory_warm_start, - verbosity=self.verbosity, - ) - - n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs - - for _ in range(n_jobs): - nth_process = len(self.opt_pros) - self.opt_pros[nth_process] = optimizer - - def _print_info(self): - print_res = PrintResults(self.opt_pros, self.verbosity) - - if self.verbosity: - for _ in range(len(self.opt_pros)): - print("") - - for results in self.results_list: - nth_process = results["nth_process"] - print_res.print_process(results, nth_process) - - def run(self, max_time: float = None): - """ - Run the optimization process with an optional maximum time limit. - - Args: - max_time (float, optional): Maximum time limit for the optimization process. Defaults to None. - """ - - self._create_shared_memory() - - for opt in self.opt_pros.values(): - opt.max_time = max_time - - self.results_list = run_search( - self.opt_pros, self.distribution, self.n_processes - ) - - self.results_ = Results(self.results_list, self.opt_pros) - - self._print_info() - - def best_para(self, id_): - """ - Retrieve the best parameters for a specific ID from the results. - - Parameters: - - id_ (int): The ID of the parameters to retrieve. - - Returns: - - Union[Dict[str, Union[int, float]], None]: The best parameters for the specified ID if found, otherwise None. - - Raises: - - ValueError: If the objective function name is not recognized. - """ - - return self.results_.best_para(id_) - - def best_score(self, id_): - """ - Return the best score for a specific ID from the results. - - Parameters: - - id_ (int): The ID for which the best score is requested. - """ - - return self.results_.best_score(id_) - - def search_data(self, id_, times=False): - """ - Retrieve search data for a specific ID from the results. Optionally exclude evaluation and iteration times if 'times' is set to False. - - Parameters: - - id_ (int): The ID of the search data to retrieve. - - times (bool, optional): Whether to exclude evaluation and iteration times. Defaults to False. - - Returns: - - pd.DataFrame: The search data for the specified ID. - """ - - search_data_ = self.results_.search_data(id_) - - if times == False: - search_data_.drop( - labels=["eval_times", "iter_times"], - axis=1, - inplace=True, - errors="ignore", - ) - return search_data_ diff --git a/src/hyperactive/optimizers/backend_stuff/run_search.py b/src/hyperactive/optimizers/backend_stuff/run_search.py deleted file mode 100644 index c9fd13e1..00000000 --- a/src/hyperactive/optimizers/backend_stuff/run_search.py +++ /dev/null @@ -1,55 +0,0 @@ -# Author: Simon Blanke -# Email: simon.blanke@yahoo.com -# License: MIT License - - -from .distribution import ( - single_process, - joblib_wrapper, - multiprocessing_wrapper, - pathos_wrapper, -) -from .process import _process_ - - -def proxy(args): - return _process_(*args) - - -dist_dict = { - "joblib": (joblib_wrapper, _process_), - "multiprocessing": (multiprocessing_wrapper, proxy), - "pathos": (pathos_wrapper, proxy), -} - - -def _get_distribution(distribution): - if hasattr(distribution, "__call__"): - return (distribution, _process_), {} - - elif isinstance(distribution, dict): - dist_key = list(distribution.keys())[0] - dist_paras = list(distribution.values())[0] - - return dist_dict[dist_key], dist_paras - - elif isinstance(distribution, str): - return dist_dict[distribution], {} - - -def run_search(searches, distribution, n_processes): - if n_processes == "auto": - n_processes = len(searches) - - searches_tuple = [(search,) for search in searches] - - if n_processes == 1: - results_list = single_process(_process_, searches) - else: - (distribution, process_func), dist_paras = _get_distribution( - distribution - ) - - results_list = distribution(process_func, searches_tuple, n_processes) - - return results_list From b376b28e16fc4f8e0193d19cdd76fc3de3a211cf Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 13 Apr 2025 10:55:08 +0200 Subject: [PATCH 57/62] add abstract class and method --- src/hyperactive/experiment/_experiment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hyperactive/experiment/_experiment.py b/src/hyperactive/experiment/_experiment.py index fa2d394f..de148327 100644 --- a/src/hyperactive/experiment/_experiment.py +++ b/src/hyperactive/experiment/_experiment.py @@ -1,10 +1,11 @@ """Base class for experiment.""" +from abc import ABC, abstractmethod from typing import Union, List, Dict, Type from skbase.base import BaseObject -class BaseExperiment(BaseObject): +class BaseExperiment(ABC, BaseObject): """Base class for experiment.""" def __init__( @@ -77,3 +78,7 @@ def _score(self, **params): Additional metadata about the search. """ raise NotImplementedError + + @abstractmethod + def objective_function(self, params): + pass From 99d98f44ada3d119ddf7890e4f9e84adb84cf199 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 13 Apr 2025 12:42:31 +0200 Subject: [PATCH 58/62] restructure to enable adaption to different optimization backends --- src/hyperactive/experiment/_experiment.py | 7 +++ .../__init__.py | 0 .../_composite_optimizer.py | 6 +- .../_optimizer_api.py | 6 +- .../gradient_free_optimizers/__init__.py | 55 +++++++++++++++++++ .../gradient_free_optimizers}/_dictionary.py | 0 .../_hyper_gradient_conv.py | 0 .../_objective_function.py | 0 .../gradient_free_optimizers}/_optimizers.py | 2 +- .../talos/__init__.py} | 0 .../optimization_backend/talos/_optimizers.py | 0 src/hyperactive/optimizers/__init__.py | 55 ------------------- .../{backend_stuff => }/_distribution.py | 0 .../{backend_stuff => }/_print_results.py | 0 .../{backend_stuff => }/_process.py | 0 .../{backend_stuff => }/_results.py | 0 src/hyperactive/optimizers/_search.py | 18 +++--- .../{strategies => pipelines}/__init__.py | 0 .../custom_optimization_strategy.py | 0 .../optimization_strategy.py | 0 .../optimizer_attributes.py | 0 .../{backend_stuff => }/search_space.py | 0 22 files changed, 78 insertions(+), 71 deletions(-) rename src/hyperactive/{memory => optimization_backend}/__init__.py (100%) rename src/hyperactive/{ => optimization_backend}/_composite_optimizer.py (88%) rename src/hyperactive/{optimizers => optimization_backend}/_optimizer_api.py (97%) create mode 100644 src/hyperactive/optimization_backend/gradient_free_optimizers/__init__.py rename src/hyperactive/{optimizers => optimization_backend/gradient_free_optimizers}/_dictionary.py (100%) rename src/hyperactive/{optimizers => optimization_backend/gradient_free_optimizers}/_hyper_gradient_conv.py (100%) rename src/hyperactive/{optimizers => optimization_backend/gradient_free_optimizers}/_objective_function.py (100%) rename src/hyperactive/{optimizers => optimization_backend/gradient_free_optimizers}/_optimizers.py (99%) rename src/hyperactive/{memory/memory.py => optimization_backend/talos/__init__.py} (100%) create mode 100644 src/hyperactive/optimization_backend/talos/_optimizers.py rename src/hyperactive/optimizers/{backend_stuff => }/_distribution.py (100%) rename src/hyperactive/optimizers/{backend_stuff => }/_print_results.py (100%) rename src/hyperactive/optimizers/{backend_stuff => }/_process.py (100%) rename src/hyperactive/optimizers/{backend_stuff => }/_results.py (100%) rename src/hyperactive/optimizers/{strategies => pipelines}/__init__.py (100%) rename src/hyperactive/optimizers/{strategies => pipelines}/custom_optimization_strategy.py (100%) rename src/hyperactive/optimizers/{strategies => pipelines}/optimization_strategy.py (100%) rename src/hyperactive/optimizers/{strategies => pipelines}/optimizer_attributes.py (100%) rename src/hyperactive/optimizers/{backend_stuff => }/search_space.py (100%) diff --git a/src/hyperactive/experiment/_experiment.py b/src/hyperactive/experiment/_experiment.py index de148327..af0fe2c4 100644 --- a/src/hyperactive/experiment/_experiment.py +++ b/src/hyperactive/experiment/_experiment.py @@ -79,6 +79,13 @@ def _score(self, **params): """ raise NotImplementedError + def backend_adapter(self, backend_adapter, s_space): + gfo_wrapper_model = backend_adapter( + experiment=self, + ) + + self.gfo_objective_function = gfo_wrapper_model(s_space()) + @abstractmethod def objective_function(self, params): pass diff --git a/src/hyperactive/memory/__init__.py b/src/hyperactive/optimization_backend/__init__.py similarity index 100% rename from src/hyperactive/memory/__init__.py rename to src/hyperactive/optimization_backend/__init__.py diff --git a/src/hyperactive/_composite_optimizer.py b/src/hyperactive/optimization_backend/_composite_optimizer.py similarity index 88% rename from src/hyperactive/_composite_optimizer.py rename to src/hyperactive/optimization_backend/_composite_optimizer.py index 3eadb82a..e1742754 100644 --- a/src/hyperactive/_composite_optimizer.py +++ b/src/hyperactive/optimization_backend/_composite_optimizer.py @@ -1,7 +1,7 @@ from typing import Union -from .optimizers.backend_stuff._distribution import run_search -from .optimizers.backend_stuff._results import Results -from .optimizers.backend_stuff._print_results import PrintResults +from ..optimizers._distribution import run_search +from ..optimizers._results import Results +from ..optimizers._print_results import PrintResults class CompositeOptimizer: diff --git a/src/hyperactive/optimizers/_optimizer_api.py b/src/hyperactive/optimization_backend/_optimizer_api.py similarity index 97% rename from src/hyperactive/optimizers/_optimizer_api.py rename to src/hyperactive/optimization_backend/_optimizer_api.py index e0b99769..e347a55e 100644 --- a/src/hyperactive/optimizers/_optimizer_api.py +++ b/src/hyperactive/optimization_backend/_optimizer_api.py @@ -4,11 +4,11 @@ import multiprocessing as mp import pandas as pd -from .backend_stuff.search_space import SearchSpace -from ._search import Search +from ..optimizers.search_space import SearchSpace +from ..optimizers._search import Search -from .._composite_optimizer import CompositeOptimizer +from ._composite_optimizer import CompositeOptimizer from skbase.base import BaseObject diff --git a/src/hyperactive/optimization_backend/gradient_free_optimizers/__init__.py b/src/hyperactive/optimization_backend/gradient_free_optimizers/__init__.py new file mode 100644 index 00000000..cd77760f --- /dev/null +++ b/src/hyperactive/optimization_backend/gradient_free_optimizers/__init__.py @@ -0,0 +1,55 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from ._optimizers import ( + HillClimbingOptimizer, + StochasticHillClimbingOptimizer, + RepulsingHillClimbingOptimizer, + SimulatedAnnealingOptimizer, + DownhillSimplexOptimizer, + RandomSearchOptimizer, + GridSearchOptimizer, + RandomRestartHillClimbingOptimizer, + RandomAnnealingOptimizer, + PowellsMethod, + PatternSearch, + ParallelTemperingOptimizer, + ParticleSwarmOptimizer, + SpiralOptimization, + GeneticAlgorithmOptimizer, + EvolutionStrategyOptimizer, + DifferentialEvolutionOptimizer, + BayesianOptimizer, + LipschitzOptimizer, + DirectAlgorithm, + TreeStructuredParzenEstimators, + ForestOptimizer, +) + + +__all__ = [ + "HillClimbingOptimizer", + "StochasticHillClimbingOptimizer", + "RepulsingHillClimbingOptimizer", + "SimulatedAnnealingOptimizer", + "DownhillSimplexOptimizer", + "RandomSearchOptimizer", + "GridSearchOptimizer", + "RandomRestartHillClimbingOptimizer", + "RandomAnnealingOptimizer", + "PowellsMethod", + "PatternSearch", + "ParallelTemperingOptimizer", + "ParticleSwarmOptimizer", + "SpiralOptimization", + "GeneticAlgorithmOptimizer", + "EvolutionStrategyOptimizer", + "DifferentialEvolutionOptimizer", + "BayesianOptimizer", + "LipschitzOptimizer", + "DirectAlgorithm", + "TreeStructuredParzenEstimators", + "ForestOptimizer", +] diff --git a/src/hyperactive/optimizers/_dictionary.py b/src/hyperactive/optimization_backend/gradient_free_optimizers/_dictionary.py similarity index 100% rename from src/hyperactive/optimizers/_dictionary.py rename to src/hyperactive/optimization_backend/gradient_free_optimizers/_dictionary.py diff --git a/src/hyperactive/optimizers/_hyper_gradient_conv.py b/src/hyperactive/optimization_backend/gradient_free_optimizers/_hyper_gradient_conv.py similarity index 100% rename from src/hyperactive/optimizers/_hyper_gradient_conv.py rename to src/hyperactive/optimization_backend/gradient_free_optimizers/_hyper_gradient_conv.py diff --git a/src/hyperactive/optimizers/_objective_function.py b/src/hyperactive/optimization_backend/gradient_free_optimizers/_objective_function.py similarity index 100% rename from src/hyperactive/optimizers/_objective_function.py rename to src/hyperactive/optimization_backend/gradient_free_optimizers/_objective_function.py diff --git a/src/hyperactive/optimizers/_optimizers.py b/src/hyperactive/optimization_backend/gradient_free_optimizers/_optimizers.py similarity index 99% rename from src/hyperactive/optimizers/_optimizers.py rename to src/hyperactive/optimization_backend/gradient_free_optimizers/_optimizers.py index 20874c89..78589961 100644 --- a/src/hyperactive/optimizers/_optimizers.py +++ b/src/hyperactive/optimization_backend/gradient_free_optimizers/_optimizers.py @@ -3,7 +3,7 @@ # License: MIT License -from ._optimizer_api import BaseOptimizer +from .._optimizer_api import BaseOptimizer from gradient_free_optimizers import ( HillClimbingOptimizer as _HillClimbingOptimizer, diff --git a/src/hyperactive/memory/memory.py b/src/hyperactive/optimization_backend/talos/__init__.py similarity index 100% rename from src/hyperactive/memory/memory.py rename to src/hyperactive/optimization_backend/talos/__init__.py diff --git a/src/hyperactive/optimization_backend/talos/_optimizers.py b/src/hyperactive/optimization_backend/talos/_optimizers.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hyperactive/optimizers/__init__.py b/src/hyperactive/optimizers/__init__.py index cd77760f..e69de29b 100644 --- a/src/hyperactive/optimizers/__init__.py +++ b/src/hyperactive/optimizers/__init__.py @@ -1,55 +0,0 @@ -# Author: Simon Blanke -# Email: simon.blanke@yahoo.com -# License: MIT License - - -from ._optimizers import ( - HillClimbingOptimizer, - StochasticHillClimbingOptimizer, - RepulsingHillClimbingOptimizer, - SimulatedAnnealingOptimizer, - DownhillSimplexOptimizer, - RandomSearchOptimizer, - GridSearchOptimizer, - RandomRestartHillClimbingOptimizer, - RandomAnnealingOptimizer, - PowellsMethod, - PatternSearch, - ParallelTemperingOptimizer, - ParticleSwarmOptimizer, - SpiralOptimization, - GeneticAlgorithmOptimizer, - EvolutionStrategyOptimizer, - DifferentialEvolutionOptimizer, - BayesianOptimizer, - LipschitzOptimizer, - DirectAlgorithm, - TreeStructuredParzenEstimators, - ForestOptimizer, -) - - -__all__ = [ - "HillClimbingOptimizer", - "StochasticHillClimbingOptimizer", - "RepulsingHillClimbingOptimizer", - "SimulatedAnnealingOptimizer", - "DownhillSimplexOptimizer", - "RandomSearchOptimizer", - "GridSearchOptimizer", - "RandomRestartHillClimbingOptimizer", - "RandomAnnealingOptimizer", - "PowellsMethod", - "PatternSearch", - "ParallelTemperingOptimizer", - "ParticleSwarmOptimizer", - "SpiralOptimization", - "GeneticAlgorithmOptimizer", - "EvolutionStrategyOptimizer", - "DifferentialEvolutionOptimizer", - "BayesianOptimizer", - "LipschitzOptimizer", - "DirectAlgorithm", - "TreeStructuredParzenEstimators", - "ForestOptimizer", -] diff --git a/src/hyperactive/optimizers/backend_stuff/_distribution.py b/src/hyperactive/optimizers/_distribution.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/_distribution.py rename to src/hyperactive/optimizers/_distribution.py diff --git a/src/hyperactive/optimizers/backend_stuff/_print_results.py b/src/hyperactive/optimizers/_print_results.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/_print_results.py rename to src/hyperactive/optimizers/_print_results.py diff --git a/src/hyperactive/optimizers/backend_stuff/_process.py b/src/hyperactive/optimizers/_process.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/_process.py rename to src/hyperactive/optimizers/_process.py diff --git a/src/hyperactive/optimizers/backend_stuff/_results.py b/src/hyperactive/optimizers/_results.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/_results.py rename to src/hyperactive/optimizers/_results.py diff --git a/src/hyperactive/optimizers/_search.py b/src/hyperactive/optimizers/_search.py index cb9d8d3a..abd65330 100644 --- a/src/hyperactive/optimizers/_search.py +++ b/src/hyperactive/optimizers/_search.py @@ -2,11 +2,16 @@ # Email: simon.blanke@yahoo.com # License: MIT License -from ._objective_function import ObjectiveFunction -from ._hyper_gradient_conv import HyperGradientConv from ._optimizer_attributes import OptimizerAttributes from ._constraint import Constraint +from ..optimization_backend.gradient_free_optimizers._objective_function import ( + ObjectiveFunction, +) +from ..optimization_backend.gradient_free_optimizers._hyper_gradient_conv import ( + HyperGradientConv, +) + class Search(OptimizerAttributes): max_time: float @@ -106,17 +111,12 @@ def _setup_process(self): def _search(self, p_bar): self._setup_process() - gfo_wrapper_model = ObjectiveFunction( - experiment=self.experiment, - ) - gfo_wrapper_model.pass_through = self.pass_through - memory_warm_start = self.hg_conv.conv_memory_warm_start(self.memory_warm_start) - gfo_objective_function = gfo_wrapper_model(self.s_space()) + self.experiment.backend_adapter(ObjectiveFunction, self.s_space) self.gfo_optimizer.init_search( - gfo_objective_function, + self.experiment.gfo_objective_function, self.n_iter, self.max_time, self.max_score, diff --git a/src/hyperactive/optimizers/strategies/__init__.py b/src/hyperactive/optimizers/pipelines/__init__.py similarity index 100% rename from src/hyperactive/optimizers/strategies/__init__.py rename to src/hyperactive/optimizers/pipelines/__init__.py diff --git a/src/hyperactive/optimizers/strategies/custom_optimization_strategy.py b/src/hyperactive/optimizers/pipelines/custom_optimization_strategy.py similarity index 100% rename from src/hyperactive/optimizers/strategies/custom_optimization_strategy.py rename to src/hyperactive/optimizers/pipelines/custom_optimization_strategy.py diff --git a/src/hyperactive/optimizers/strategies/optimization_strategy.py b/src/hyperactive/optimizers/pipelines/optimization_strategy.py similarity index 100% rename from src/hyperactive/optimizers/strategies/optimization_strategy.py rename to src/hyperactive/optimizers/pipelines/optimization_strategy.py diff --git a/src/hyperactive/optimizers/strategies/optimizer_attributes.py b/src/hyperactive/optimizers/pipelines/optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimizers/strategies/optimizer_attributes.py rename to src/hyperactive/optimizers/pipelines/optimizer_attributes.py diff --git a/src/hyperactive/optimizers/backend_stuff/search_space.py b/src/hyperactive/optimizers/search_space.py similarity index 100% rename from src/hyperactive/optimizers/backend_stuff/search_space.py rename to src/hyperactive/optimizers/search_space.py From 18397d0eb3fdb03e05599251296d6714653f57d9 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 27 Apr 2025 12:17:52 +0200 Subject: [PATCH 59/62] move files into optimization path --- .../{optimization_backend => optimization}/__init__.py | 0 .../_composite_optimizer.py | 0 src/hyperactive/{optimizers => optimization}/_constraint.py | 0 src/hyperactive/{optimizers => optimization}/_distribution.py | 0 .../{optimization_backend => optimization}/_optimizer_api.py | 0 .../{optimizers => optimization}/_optimizer_attributes.py | 0 src/hyperactive/{optimizers => optimization}/_print_results.py | 0 src/hyperactive/{optimizers => optimization}/_process.py | 0 src/hyperactive/{optimizers => optimization}/_results.py | 0 src/hyperactive/{optimizers => optimization}/_search.py | 0 .../gradient_free_optimizers/__init__.py | 0 .../gradient_free_optimizers/_dictionary.py | 0 .../gradient_free_optimizers/_hyper_gradient_conv.py | 0 .../gradient_free_optimizers/_objective_function.py | 0 .../gradient_free_optimizers/_optimizers.py | 0 .../{optimizers => optimization}/pipelines/__init__.py | 0 .../pipelines/custom_optimization_strategy.py | 0 .../pipelines/optimization_strategy.py | 0 .../pipelines/optimizer_attributes.py | 0 src/hyperactive/{optimizers => optimization}/search_space.py | 0 .../{optimization_backend => optimization}/talos/__init__.py | 0 .../{optimization_backend => optimization}/talos/_optimizers.py | 0 src/hyperactive/optimizers/__init__.py | 0 23 files changed, 0 insertions(+), 0 deletions(-) rename src/hyperactive/{optimization_backend => optimization}/__init__.py (100%) rename src/hyperactive/{optimization_backend => optimization}/_composite_optimizer.py (100%) rename src/hyperactive/{optimizers => optimization}/_constraint.py (100%) rename src/hyperactive/{optimizers => optimization}/_distribution.py (100%) rename src/hyperactive/{optimization_backend => optimization}/_optimizer_api.py (100%) rename src/hyperactive/{optimizers => optimization}/_optimizer_attributes.py (100%) rename src/hyperactive/{optimizers => optimization}/_print_results.py (100%) rename src/hyperactive/{optimizers => optimization}/_process.py (100%) rename src/hyperactive/{optimizers => optimization}/_results.py (100%) rename src/hyperactive/{optimizers => optimization}/_search.py (100%) rename src/hyperactive/{optimization_backend => optimization}/gradient_free_optimizers/__init__.py (100%) rename src/hyperactive/{optimization_backend => optimization}/gradient_free_optimizers/_dictionary.py (100%) rename src/hyperactive/{optimization_backend => optimization}/gradient_free_optimizers/_hyper_gradient_conv.py (100%) rename src/hyperactive/{optimization_backend => optimization}/gradient_free_optimizers/_objective_function.py (100%) rename src/hyperactive/{optimization_backend => optimization}/gradient_free_optimizers/_optimizers.py (100%) rename src/hyperactive/{optimizers => optimization}/pipelines/__init__.py (100%) rename src/hyperactive/{optimizers => optimization}/pipelines/custom_optimization_strategy.py (100%) rename src/hyperactive/{optimizers => optimization}/pipelines/optimization_strategy.py (100%) rename src/hyperactive/{optimizers => optimization}/pipelines/optimizer_attributes.py (100%) rename src/hyperactive/{optimizers => optimization}/search_space.py (100%) rename src/hyperactive/{optimization_backend => optimization}/talos/__init__.py (100%) rename src/hyperactive/{optimization_backend => optimization}/talos/_optimizers.py (100%) delete mode 100644 src/hyperactive/optimizers/__init__.py diff --git a/src/hyperactive/optimization_backend/__init__.py b/src/hyperactive/optimization/__init__.py similarity index 100% rename from src/hyperactive/optimization_backend/__init__.py rename to src/hyperactive/optimization/__init__.py diff --git a/src/hyperactive/optimization_backend/_composite_optimizer.py b/src/hyperactive/optimization/_composite_optimizer.py similarity index 100% rename from src/hyperactive/optimization_backend/_composite_optimizer.py rename to src/hyperactive/optimization/_composite_optimizer.py diff --git a/src/hyperactive/optimizers/_constraint.py b/src/hyperactive/optimization/_constraint.py similarity index 100% rename from src/hyperactive/optimizers/_constraint.py rename to src/hyperactive/optimization/_constraint.py diff --git a/src/hyperactive/optimizers/_distribution.py b/src/hyperactive/optimization/_distribution.py similarity index 100% rename from src/hyperactive/optimizers/_distribution.py rename to src/hyperactive/optimization/_distribution.py diff --git a/src/hyperactive/optimization_backend/_optimizer_api.py b/src/hyperactive/optimization/_optimizer_api.py similarity index 100% rename from src/hyperactive/optimization_backend/_optimizer_api.py rename to src/hyperactive/optimization/_optimizer_api.py diff --git a/src/hyperactive/optimizers/_optimizer_attributes.py b/src/hyperactive/optimization/_optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimizers/_optimizer_attributes.py rename to src/hyperactive/optimization/_optimizer_attributes.py diff --git a/src/hyperactive/optimizers/_print_results.py b/src/hyperactive/optimization/_print_results.py similarity index 100% rename from src/hyperactive/optimizers/_print_results.py rename to src/hyperactive/optimization/_print_results.py diff --git a/src/hyperactive/optimizers/_process.py b/src/hyperactive/optimization/_process.py similarity index 100% rename from src/hyperactive/optimizers/_process.py rename to src/hyperactive/optimization/_process.py diff --git a/src/hyperactive/optimizers/_results.py b/src/hyperactive/optimization/_results.py similarity index 100% rename from src/hyperactive/optimizers/_results.py rename to src/hyperactive/optimization/_results.py diff --git a/src/hyperactive/optimizers/_search.py b/src/hyperactive/optimization/_search.py similarity index 100% rename from src/hyperactive/optimizers/_search.py rename to src/hyperactive/optimization/_search.py diff --git a/src/hyperactive/optimization_backend/gradient_free_optimizers/__init__.py b/src/hyperactive/optimization/gradient_free_optimizers/__init__.py similarity index 100% rename from src/hyperactive/optimization_backend/gradient_free_optimizers/__init__.py rename to src/hyperactive/optimization/gradient_free_optimizers/__init__.py diff --git a/src/hyperactive/optimization_backend/gradient_free_optimizers/_dictionary.py b/src/hyperactive/optimization/gradient_free_optimizers/_dictionary.py similarity index 100% rename from src/hyperactive/optimization_backend/gradient_free_optimizers/_dictionary.py rename to src/hyperactive/optimization/gradient_free_optimizers/_dictionary.py diff --git a/src/hyperactive/optimization_backend/gradient_free_optimizers/_hyper_gradient_conv.py b/src/hyperactive/optimization/gradient_free_optimizers/_hyper_gradient_conv.py similarity index 100% rename from src/hyperactive/optimization_backend/gradient_free_optimizers/_hyper_gradient_conv.py rename to src/hyperactive/optimization/gradient_free_optimizers/_hyper_gradient_conv.py diff --git a/src/hyperactive/optimization_backend/gradient_free_optimizers/_objective_function.py b/src/hyperactive/optimization/gradient_free_optimizers/_objective_function.py similarity index 100% rename from src/hyperactive/optimization_backend/gradient_free_optimizers/_objective_function.py rename to src/hyperactive/optimization/gradient_free_optimizers/_objective_function.py diff --git a/src/hyperactive/optimization_backend/gradient_free_optimizers/_optimizers.py b/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py similarity index 100% rename from src/hyperactive/optimization_backend/gradient_free_optimizers/_optimizers.py rename to src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py diff --git a/src/hyperactive/optimizers/pipelines/__init__.py b/src/hyperactive/optimization/pipelines/__init__.py similarity index 100% rename from src/hyperactive/optimizers/pipelines/__init__.py rename to src/hyperactive/optimization/pipelines/__init__.py diff --git a/src/hyperactive/optimizers/pipelines/custom_optimization_strategy.py b/src/hyperactive/optimization/pipelines/custom_optimization_strategy.py similarity index 100% rename from src/hyperactive/optimizers/pipelines/custom_optimization_strategy.py rename to src/hyperactive/optimization/pipelines/custom_optimization_strategy.py diff --git a/src/hyperactive/optimizers/pipelines/optimization_strategy.py b/src/hyperactive/optimization/pipelines/optimization_strategy.py similarity index 100% rename from src/hyperactive/optimizers/pipelines/optimization_strategy.py rename to src/hyperactive/optimization/pipelines/optimization_strategy.py diff --git a/src/hyperactive/optimizers/pipelines/optimizer_attributes.py b/src/hyperactive/optimization/pipelines/optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimizers/pipelines/optimizer_attributes.py rename to src/hyperactive/optimization/pipelines/optimizer_attributes.py diff --git a/src/hyperactive/optimizers/search_space.py b/src/hyperactive/optimization/search_space.py similarity index 100% rename from src/hyperactive/optimizers/search_space.py rename to src/hyperactive/optimization/search_space.py diff --git a/src/hyperactive/optimization_backend/talos/__init__.py b/src/hyperactive/optimization/talos/__init__.py similarity index 100% rename from src/hyperactive/optimization_backend/talos/__init__.py rename to src/hyperactive/optimization/talos/__init__.py diff --git a/src/hyperactive/optimization_backend/talos/_optimizers.py b/src/hyperactive/optimization/talos/_optimizers.py similarity index 100% rename from src/hyperactive/optimization_backend/talos/_optimizers.py rename to src/hyperactive/optimization/talos/_optimizers.py diff --git a/src/hyperactive/optimizers/__init__.py b/src/hyperactive/optimizers/__init__.py deleted file mode 100644 index e69de29b..00000000 From 6433801727b657b6531d5a20dc48a573206c811e Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 27 Apr 2025 12:20:09 +0200 Subject: [PATCH 60/62] adapt paths to new structure --- src/hyperactive/optimization/_composite_optimizer.py | 6 +++--- src/hyperactive/optimization/_optimizer_api.py | 4 ++-- src/hyperactive/optimization/_search.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hyperactive/optimization/_composite_optimizer.py b/src/hyperactive/optimization/_composite_optimizer.py index e1742754..3d6fa60b 100644 --- a/src/hyperactive/optimization/_composite_optimizer.py +++ b/src/hyperactive/optimization/_composite_optimizer.py @@ -1,7 +1,7 @@ from typing import Union -from ..optimizers._distribution import run_search -from ..optimizers._results import Results -from ..optimizers._print_results import PrintResults +from ._distribution import run_search +from ._results import Results +from ._print_results import PrintResults class CompositeOptimizer: diff --git a/src/hyperactive/optimization/_optimizer_api.py b/src/hyperactive/optimization/_optimizer_api.py index e347a55e..86b19b8b 100644 --- a/src/hyperactive/optimization/_optimizer_api.py +++ b/src/hyperactive/optimization/_optimizer_api.py @@ -4,8 +4,8 @@ import multiprocessing as mp import pandas as pd -from ..optimizers.search_space import SearchSpace -from ..optimizers._search import Search +from .search_space import SearchSpace +from ._search import Search from ._composite_optimizer import CompositeOptimizer diff --git a/src/hyperactive/optimization/_search.py b/src/hyperactive/optimization/_search.py index abd65330..87e732c8 100644 --- a/src/hyperactive/optimization/_search.py +++ b/src/hyperactive/optimization/_search.py @@ -5,10 +5,10 @@ from ._optimizer_attributes import OptimizerAttributes from ._constraint import Constraint -from ..optimization_backend.gradient_free_optimizers._objective_function import ( +from .gradient_free_optimizers._objective_function import ( ObjectiveFunction, ) -from ..optimization_backend.gradient_free_optimizers._hyper_gradient_conv import ( +from .gradient_free_optimizers._hyper_gradient_conv import ( HyperGradientConv, ) From 3b565eedc8cf6a81f55e9cd57ea6833d0cb75106 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 27 Apr 2025 13:18:38 +0200 Subject: [PATCH 61/62] continue restructuring... --- .../optimization/_composite_optimizer.py | 9 ++ .../optimization/_optimizer_api.py | 30 +++-- src/hyperactive/optimization/_run_info.py | 9 ++ src/hyperactive/optimization/_search.py | 113 ++++++------------ src/hyperactive/optimization/_search_info.py | 15 +++ .../gradient_free_optimizers/_runner.py | 96 +++++++++++++++ .../adapter/__init__.py | 13 ++ .../adapter/_adapter.py | 10 ++ .../{ => adapter}/_dictionary.py | 0 .../{ => adapter}/_hyper_gradient_conv.py | 0 .../{ => adapter}/_objective_function.py | 0 .../adapter/_results.py | 3 + 12 files changed, 208 insertions(+), 90 deletions(-) create mode 100644 src/hyperactive/optimization/_run_info.py create mode 100644 src/hyperactive/optimization/_search_info.py create mode 100644 src/hyperactive/optimization/gradient_free_optimizers/_runner.py create mode 100644 src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py create mode 100644 src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py rename src/hyperactive/optimization/gradient_free_optimizers/{ => adapter}/_dictionary.py (100%) rename src/hyperactive/optimization/gradient_free_optimizers/{ => adapter}/_hyper_gradient_conv.py (100%) rename src/hyperactive/optimization/gradient_free_optimizers/{ => adapter}/_objective_function.py (100%) create mode 100644 src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py diff --git a/src/hyperactive/optimization/_composite_optimizer.py b/src/hyperactive/optimization/_composite_optimizer.py index 3d6fa60b..bb174557 100644 --- a/src/hyperactive/optimization/_composite_optimizer.py +++ b/src/hyperactive/optimization/_composite_optimizer.py @@ -3,6 +3,8 @@ from ._results import Results from ._print_results import PrintResults +from ._run_info import RunInfo + class CompositeOptimizer: optimizers: list @@ -24,6 +26,13 @@ def run( if not verbosity: verbosity = [] + run_info = RunInfo( + max_time, + distribution, + n_processes, + verbosity, + ) + self.collected_searches = [] for optimizer in self.optimizers: self.collected_searches += optimizer.searches diff --git a/src/hyperactive/optimization/_optimizer_api.py b/src/hyperactive/optimization/_optimizer_api.py index 86b19b8b..baaed60d 100644 --- a/src/hyperactive/optimization/_optimizer_api.py +++ b/src/hyperactive/optimization/_optimizer_api.py @@ -7,6 +7,8 @@ from .search_space import SearchSpace from ._search import Search +from ._search_info import SearchInfo + from ._composite_optimizer import CompositeOptimizer @@ -99,21 +101,22 @@ def add_search( n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs + search_info = SearchInfo( + experiment, + s_space, + n_iter, + initialize, + constraints, + max_score, + early_stopping, + random_state, + memory, + memory_warm_start, + ) + for _ in range(n_jobs): search = Search(self.optimizer_class, self.opt_params) - search.setup( - experiment=experiment, - s_space=s_space, - n_iter=n_iter, - initialize=initialize, - constraints=constraints, - pass_through=pass_through, - max_score=max_score, - early_stopping=early_stopping, - random_state=random_state, - memory=memory, - memory_warm_start=memory_warm_start, - ) + search.setup(search_info) self.searches.append(search) @property @@ -130,6 +133,7 @@ def run( n_processes: Union[str, int] = "auto", verbosity: list = ["progress_bar", "print_results", "print_times"], ): + self.comp_opt = CompositeOptimizer(self) self.comp_opt.run(max_time, distribution, n_processes, verbosity) diff --git a/src/hyperactive/optimization/_run_info.py b/src/hyperactive/optimization/_run_info.py new file mode 100644 index 00000000..9bfd6352 --- /dev/null +++ b/src/hyperactive/optimization/_run_info.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass() +class RunInfo: + max_time: int + distribution: int + n_processes: int + verbosity: int diff --git a/src/hyperactive/optimization/_search.py b/src/hyperactive/optimization/_search.py index 87e732c8..39eb4ad9 100644 --- a/src/hyperactive/optimization/_search.py +++ b/src/hyperactive/optimization/_search.py @@ -5,12 +5,11 @@ from ._optimizer_attributes import OptimizerAttributes from ._constraint import Constraint -from .gradient_free_optimizers._objective_function import ( - ObjectiveFunction, -) -from .gradient_free_optimizers._hyper_gradient_conv import ( +from .gradient_free_optimizers.adapter._hyper_gradient_conv import ( HyperGradientConv, ) +from .gradient_free_optimizers.adapter import Adapter +from .gradient_free_optimizers._runner import Runner class Search(OptimizerAttributes): @@ -22,32 +21,24 @@ def __init__(self, optimizer_class, opt_params): self.optimizer_class = optimizer_class self.opt_params = opt_params - def setup( - self, - experiment, - s_space, - n_iter, - initialize, - constraints, - pass_through, - max_score, - early_stopping, - random_state, - memory, - memory_warm_start, - ): - self.experiment = experiment - self.s_space = s_space - self.n_iter = n_iter - - self.initialize = initialize - self.constraints = constraints - self.pass_through = pass_through - self.max_score = max_score - self.early_stopping = early_stopping - self.random_state = random_state - self.memory = memory - self.memory_warm_start = memory_warm_start + self.runner = Runner(optimizer_class, opt_params) + + def setup(self, search_info): + self.search_info = search_info + + self.adapter = Adapter(search_info) + + self.experiment = search_info.experiment + self.s_space = search_info.s_space + self.n_iter = search_info.n_iter + + self.initialize = search_info.initialize + self.constraints = search_info.constraints + self.max_score = search_info.max_score + self.early_stopping = search_info.early_stopping + self.random_state = search_info.random_state + self.memory = search_info.memory + self.memory_warm_start = search_info.memory_warm_start def pass_args(self, max_time, nth_process, verbosity): self.max_time = max_time @@ -58,45 +49,10 @@ def pass_args(self, max_time, nth_process, verbosity): else: self.verbosity = [] - def convert_results2hyper(self): - self.eval_times = sum(self.gfo_optimizer.eval_times) - self.iter_times = sum(self.gfo_optimizer.iter_times) - - if self.gfo_optimizer.best_para is not None: - value = self.hg_conv.para2value(self.gfo_optimizer.best_para) - position = self.hg_conv.position2value(value) - best_para = self.hg_conv.value2para(position) - self.best_para = best_para - else: - self.best_para = None - - self.best_score = self.gfo_optimizer.best_score - self.positions = self.gfo_optimizer.search_data - self.search_data = self.hg_conv.positions2results(self.positions) - - results_dd = self.gfo_optimizer.search_data.drop_duplicates( - subset=self.s_space.dim_keys, keep="first" - ) - self.memory_values_df = results_dd[ - self.s_space.dim_keys + ["score"] - ].reset_index(drop=True) - def _setup_process(self): self.hg_conv = HyperGradientConv(self.s_space) - initialize = self.hg_conv.conv_initialize(self.initialize) - search_space_positions = self.s_space.positions - - # conv warm start for smbo from values into positions - if "warm_start_smbo" in self.opt_params: - self.opt_params["warm_start_smbo"] = self.hg_conv.conv_memory_warm_start( - self.opt_params["warm_start_smbo"] - ) - - gfo_constraints = [ - Constraint(constraint, self.s_space) for constraint in self.constraints - ] - + """ self.gfo_optimizer = self.optimizer_class( search_space=search_space_positions, initialize=initialize, @@ -105,16 +61,18 @@ def _setup_process(self): nth_process=self.nth_process, **self.opt_params, ) - - self.conv = self.gfo_optimizer.conv + """ + # self.conv = self.gfo_optimizer.conv def _search(self, p_bar): self._setup_process() memory_warm_start = self.hg_conv.conv_memory_warm_start(self.memory_warm_start) - self.experiment.backend_adapter(ObjectiveFunction, self.s_space) + self.experiment.backend_adapter(self.adapter.objective_function, self.s_space) + self.runner.run_search(self.search_info, self.nth_process, self.max_time, p_bar) + """ self.gfo_optimizer.init_search( self.experiment.gfo_objective_function, self.n_iter, @@ -152,15 +110,16 @@ def _search(self, p_bar): p_bar.refresh() self.gfo_optimizer.finish_search() + """ - self.convert_results2hyper() + self.runner.convert_results2hyper() self._add_result_attributes( - self.best_para, - self.best_score, - self.gfo_optimizer.p_bar._best_since_iter, - self.eval_times, - self.iter_times, - self.search_data, - self.gfo_optimizer.random_seed, + self.runner.best_para, + self.runner.best_score, + self.runner.gfo_optimizer.p_bar._best_since_iter, + self.runner.eval_times, + self.runner.iter_times, + self.runner.search_data, + self.runner.gfo_optimizer.random_seed, ) diff --git a/src/hyperactive/optimization/_search_info.py b/src/hyperactive/optimization/_search_info.py new file mode 100644 index 00000000..23c6b623 --- /dev/null +++ b/src/hyperactive/optimization/_search_info.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass() +class SearchInfo: + experiment: int + s_space: int + n_iter: int + initialize: int + constraints: int + max_score: int + early_stopping: int + random_state: int + memory: int + memory_warm_start: int diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_runner.py b/src/hyperactive/optimization/gradient_free_optimizers/_runner.py new file mode 100644 index 00000000..c164fa78 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_runner.py @@ -0,0 +1,96 @@ +from .adapter._hyper_gradient_conv import HyperGradientConv + + +class Runner: + def __init__(self, optimizer_class, opt_params): + self.optimizer_class = optimizer_class + self.opt_params = opt_params + + def run_search(self, search_info, nth_process, max_time, p_bar): + self.search_info = search_info + + self.hg_conv = HyperGradientConv(search_info.s_space) + + search_space_positions = search_info.s_space.positions + initialize = self.hg_conv.conv_initialize(search_info.initialize) + + # conv warm start for smbo from values into positions + if "warm_start_smbo" in self.opt_params: + self.opt_params["warm_start_smbo"] = self.hg_conv.conv_memory_warm_start( + self.opt_params["warm_start_smbo"] + ) + + gfo_constraints = [ + Constraint(constraint, self.s_space) + for constraint in search_info.constraints + ] + + self.gfo_optimizer = self.optimizer_class( + search_space=search_space_positions, + initialize=initialize, + constraints=gfo_constraints, + random_state=search_info.random_state, + nth_process=nth_process, + **self.opt_params, + ) + + self.gfo_optimizer.init_search( + search_info.experiment.gfo_objective_function, + search_info.n_iter, + max_time, + search_info.max_score, + search_info.early_stopping, + search_info.memory, + search_info.memory_warm_start, + False, + ) + for nth_iter in range(search_info.n_iter): + if p_bar: + p_bar.set_description( + "[" + + str(nth_process) + + "] " + + str(search_info.experiment.__class__.__name__) + + " (" + + self.optimizer_class.name + + ")", + ) + + self.gfo_optimizer.search_step(nth_iter) + if self.gfo_optimizer.stop.check(): + break + + if p_bar: + p_bar.set_postfix( + best_score=str(self.gfo_optimizer.score_best), + best_pos=str(self.gfo_optimizer.pos_best), + best_iter=str(self.gfo_optimizer.p_bar._best_since_iter), + ) + + p_bar.update(1) + p_bar.refresh() + + self.gfo_optimizer.finish_search() + + def convert_results2hyper(self): + self.eval_times = sum(self.gfo_optimizer.eval_times) + self.iter_times = sum(self.gfo_optimizer.iter_times) + + if self.gfo_optimizer.best_para is not None: + value = self.hg_conv.para2value(self.gfo_optimizer.best_para) + position = self.hg_conv.position2value(value) + best_para = self.hg_conv.value2para(position) + self.best_para = best_para + else: + self.best_para = None + + self.best_score = self.gfo_optimizer.best_score + self.positions = self.gfo_optimizer.search_data + self.search_data = self.hg_conv.positions2results(self.positions) + + results_dd = self.gfo_optimizer.search_data.drop_duplicates( + subset=self.search_info.s_space.dim_keys, keep="first" + ) + self.memory_values_df = results_dd[ + self.search_info.s_space.dim_keys + ["score"] + ].reset_index(drop=True) diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py new file mode 100644 index 00000000..ce9326f2 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py @@ -0,0 +1,13 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from ._adapter import ( + Adapter, +) + + +__all__ = [ + "Adapter", +] diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py new file mode 100644 index 00000000..d28c4fcf --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py @@ -0,0 +1,10 @@ +from ._objective_function import ObjectiveFunction +from ._hyper_gradient_conv import HyperGradientConv +from ._results import Results + + +class Adapter: + def __init__(self, search_info): + self.objective_function = ObjectiveFunction + self.search_space = HyperGradientConv + self.results = Results diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_dictionary.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_dictionary.py similarity index 100% rename from src/hyperactive/optimization/gradient_free_optimizers/_dictionary.py rename to src/hyperactive/optimization/gradient_free_optimizers/adapter/_dictionary.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_hyper_gradient_conv.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_hyper_gradient_conv.py similarity index 100% rename from src/hyperactive/optimization/gradient_free_optimizers/_hyper_gradient_conv.py rename to src/hyperactive/optimization/gradient_free_optimizers/adapter/_hyper_gradient_conv.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_objective_function.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_objective_function.py similarity index 100% rename from src/hyperactive/optimization/gradient_free_optimizers/_objective_function.py rename to src/hyperactive/optimization/gradient_free_optimizers/adapter/_objective_function.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py new file mode 100644 index 00000000..51be0ab8 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py @@ -0,0 +1,3 @@ +class Results: + def __init__(self): + pass From be8c97f84f86195f44079cf9ebf59a003d2da6c8 Mon Sep 17 00:00:00 2001 From: Simon Blanke Date: Sun, 27 Apr 2025 15:52:12 +0200 Subject: [PATCH 62/62] continue restructuring... --- .../optimization/_abstract_optimizer.py | 16 +++++ .../_base_optimizer.py} | 4 +- .../_composite_optimizer.py | 0 .../_constraint.py | 0 .../_distribution.py | 0 .../_optimizer_attributes.py | 0 .../gradient_free_optimizers/_optimizers.py | 2 +- .../_print_results.py | 0 .../_process.py | 0 .../_results.py | 0 .../_run_info.py | 0 .../gradient_free_optimizers/_runner.py | 6 +- .../{ => gradient_free_optimizers}/_search.py | 64 +------------------ .../_search_info.py | 0 .../search_space.py | 0 .../optimization/talos/__init__.py | 1 + .../optimization/talos/_optimizers.py | 12 ++++ 17 files changed, 40 insertions(+), 65 deletions(-) create mode 100644 src/hyperactive/optimization/_abstract_optimizer.py rename src/hyperactive/optimization/{_optimizer_api.py => gradient_free_optimizers/_base_optimizer.py} (98%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_composite_optimizer.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_constraint.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_distribution.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_optimizer_attributes.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_print_results.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_process.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_results.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_run_info.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_search.py (50%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/_search_info.py (100%) rename src/hyperactive/optimization/{ => gradient_free_optimizers}/search_space.py (100%) diff --git a/src/hyperactive/optimization/_abstract_optimizer.py b/src/hyperactive/optimization/_abstract_optimizer.py new file mode 100644 index 00000000..eefef594 --- /dev/null +++ b/src/hyperactive/optimization/_abstract_optimizer.py @@ -0,0 +1,16 @@ +from abc import ABC, abstractmethod +from typing import Union, List, Dict, Type +from skbase.base import BaseObject + + +class AbstractOptimizer(ABC, BaseObject): + def __init__(self, *args, **kwargs): + pass + + @abstractmethod + def add_search(self, *args, **kwargs): + pass + + @abstractmethod + def run(self, *args, **kwargs): + pass diff --git a/src/hyperactive/optimization/_optimizer_api.py b/src/hyperactive/optimization/gradient_free_optimizers/_base_optimizer.py similarity index 98% rename from src/hyperactive/optimization/_optimizer_api.py rename to src/hyperactive/optimization/gradient_free_optimizers/_base_optimizer.py index baaed60d..6b07cd2f 100644 --- a/src/hyperactive/optimization/_optimizer_api.py +++ b/src/hyperactive/optimization/gradient_free_optimizers/_base_optimizer.py @@ -12,10 +12,10 @@ from ._composite_optimizer import CompositeOptimizer -from skbase.base import BaseObject +from .._abstract_optimizer import AbstractOptimizer -class BaseOptimizer(BaseObject): +class BaseOptimizer(AbstractOptimizer): """Base class for optimizer.""" n_search: int diff --git a/src/hyperactive/optimization/_composite_optimizer.py b/src/hyperactive/optimization/gradient_free_optimizers/_composite_optimizer.py similarity index 100% rename from src/hyperactive/optimization/_composite_optimizer.py rename to src/hyperactive/optimization/gradient_free_optimizers/_composite_optimizer.py diff --git a/src/hyperactive/optimization/_constraint.py b/src/hyperactive/optimization/gradient_free_optimizers/_constraint.py similarity index 100% rename from src/hyperactive/optimization/_constraint.py rename to src/hyperactive/optimization/gradient_free_optimizers/_constraint.py diff --git a/src/hyperactive/optimization/_distribution.py b/src/hyperactive/optimization/gradient_free_optimizers/_distribution.py similarity index 100% rename from src/hyperactive/optimization/_distribution.py rename to src/hyperactive/optimization/gradient_free_optimizers/_distribution.py diff --git a/src/hyperactive/optimization/_optimizer_attributes.py b/src/hyperactive/optimization/gradient_free_optimizers/_optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimization/_optimizer_attributes.py rename to src/hyperactive/optimization/gradient_free_optimizers/_optimizer_attributes.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py b/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py index 78589961..145881b2 100644 --- a/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py +++ b/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py @@ -3,7 +3,7 @@ # License: MIT License -from .._optimizer_api import BaseOptimizer +from ._base_optimizer import BaseOptimizer from gradient_free_optimizers import ( HillClimbingOptimizer as _HillClimbingOptimizer, diff --git a/src/hyperactive/optimization/_print_results.py b/src/hyperactive/optimization/gradient_free_optimizers/_print_results.py similarity index 100% rename from src/hyperactive/optimization/_print_results.py rename to src/hyperactive/optimization/gradient_free_optimizers/_print_results.py diff --git a/src/hyperactive/optimization/_process.py b/src/hyperactive/optimization/gradient_free_optimizers/_process.py similarity index 100% rename from src/hyperactive/optimization/_process.py rename to src/hyperactive/optimization/gradient_free_optimizers/_process.py diff --git a/src/hyperactive/optimization/_results.py b/src/hyperactive/optimization/gradient_free_optimizers/_results.py similarity index 100% rename from src/hyperactive/optimization/_results.py rename to src/hyperactive/optimization/gradient_free_optimizers/_results.py diff --git a/src/hyperactive/optimization/_run_info.py b/src/hyperactive/optimization/gradient_free_optimizers/_run_info.py similarity index 100% rename from src/hyperactive/optimization/_run_info.py rename to src/hyperactive/optimization/gradient_free_optimizers/_run_info.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_runner.py b/src/hyperactive/optimization/gradient_free_optimizers/_runner.py index c164fa78..f658ddca 100644 --- a/src/hyperactive/optimization/gradient_free_optimizers/_runner.py +++ b/src/hyperactive/optimization/gradient_free_optimizers/_runner.py @@ -25,6 +25,10 @@ def run_search(self, search_info, nth_process, max_time, p_bar): for constraint in search_info.constraints ] + memory_warm_start = self.hg_conv.conv_memory_warm_start( + search_info.memory_warm_start + ) + self.gfo_optimizer = self.optimizer_class( search_space=search_space_positions, initialize=initialize, @@ -41,7 +45,7 @@ def run_search(self, search_info, nth_process, max_time, p_bar): search_info.max_score, search_info.early_stopping, search_info.memory, - search_info.memory_warm_start, + memory_warm_start, False, ) for nth_iter in range(search_info.n_iter): diff --git a/src/hyperactive/optimization/_search.py b/src/hyperactive/optimization/gradient_free_optimizers/_search.py similarity index 50% rename from src/hyperactive/optimization/_search.py rename to src/hyperactive/optimization/gradient_free_optimizers/_search.py index 39eb4ad9..7cf1621e 100644 --- a/src/hyperactive/optimization/_search.py +++ b/src/hyperactive/optimization/gradient_free_optimizers/_search.py @@ -5,11 +5,11 @@ from ._optimizer_attributes import OptimizerAttributes from ._constraint import Constraint -from .gradient_free_optimizers.adapter._hyper_gradient_conv import ( +from .adapter._hyper_gradient_conv import ( HyperGradientConv, ) -from .gradient_free_optimizers.adapter import Adapter -from .gradient_free_optimizers._runner import Runner +from .adapter import Adapter +from ._runner import Runner class Search(OptimizerAttributes): @@ -49,68 +49,10 @@ def pass_args(self, max_time, nth_process, verbosity): else: self.verbosity = [] - def _setup_process(self): - self.hg_conv = HyperGradientConv(self.s_space) - - """ - self.gfo_optimizer = self.optimizer_class( - search_space=search_space_positions, - initialize=initialize, - constraints=gfo_constraints, - random_state=self.random_state, - nth_process=self.nth_process, - **self.opt_params, - ) - """ - # self.conv = self.gfo_optimizer.conv - def _search(self, p_bar): - self._setup_process() - - memory_warm_start = self.hg_conv.conv_memory_warm_start(self.memory_warm_start) - self.experiment.backend_adapter(self.adapter.objective_function, self.s_space) self.runner.run_search(self.search_info, self.nth_process, self.max_time, p_bar) - """ - self.gfo_optimizer.init_search( - self.experiment.gfo_objective_function, - self.n_iter, - self.max_time, - self.max_score, - self.early_stopping, - self.memory, - memory_warm_start, - False, - ) - for nth_iter in range(self.n_iter): - if p_bar: - p_bar.set_description( - "[" - + str(self.nth_process) - + "] " - + str(self.experiment.__class__.__name__) - + " (" - + self.optimizer_class.name - + ")", - ) - - self.gfo_optimizer.search_step(nth_iter) - if self.gfo_optimizer.stop.check(): - break - - if p_bar: - p_bar.set_postfix( - best_score=str(self.gfo_optimizer.score_best), - best_pos=str(self.gfo_optimizer.pos_best), - best_iter=str(self.gfo_optimizer.p_bar._best_since_iter), - ) - - p_bar.update(1) - p_bar.refresh() - - self.gfo_optimizer.finish_search() - """ self.runner.convert_results2hyper() diff --git a/src/hyperactive/optimization/_search_info.py b/src/hyperactive/optimization/gradient_free_optimizers/_search_info.py similarity index 100% rename from src/hyperactive/optimization/_search_info.py rename to src/hyperactive/optimization/gradient_free_optimizers/_search_info.py diff --git a/src/hyperactive/optimization/search_space.py b/src/hyperactive/optimization/gradient_free_optimizers/search_space.py similarity index 100% rename from src/hyperactive/optimization/search_space.py rename to src/hyperactive/optimization/gradient_free_optimizers/search_space.py diff --git a/src/hyperactive/optimization/talos/__init__.py b/src/hyperactive/optimization/talos/__init__.py index e69de29b..344704b4 100644 --- a/src/hyperactive/optimization/talos/__init__.py +++ b/src/hyperactive/optimization/talos/__init__.py @@ -0,0 +1 @@ +from ._optimizers import TalosOptimizer diff --git a/src/hyperactive/optimization/talos/_optimizers.py b/src/hyperactive/optimization/talos/_optimizers.py index e69de29b..5c98294b 100644 --- a/src/hyperactive/optimization/talos/_optimizers.py +++ b/src/hyperactive/optimization/talos/_optimizers.py @@ -0,0 +1,12 @@ +from .._abstract_optimizer import AbstractOptimizer + + +class TalosOptimizer(AbstractOptimizer): + def __init__(self): + pass + + def add_search(self): + pass + + def run(self): + pass