diff --git a/examples/v5_API_example/example.py b/examples/v5_API_example/example.py new file mode 100644 index 00000000..ebe98b6d --- /dev/null +++ b/examples/v5_API_example/example.py @@ -0,0 +1,39 @@ +import numpy as np +from sklearn.datasets import load_diabetes +from sklearn.tree import DecisionTreeRegressor +from sklearn.model_selection import cross_val_score + +from hyperactive.search_config import SearchConfig +from hyperactive.optimizers import HillClimbingOptimizer + +from hyperactive.base import BaseExperiment + + +class SklearnExperiment(BaseExperiment): + def setup(self, estimator, X, y, cv=5): + self.estimator = estimator + self.X = X + self.y = y + self.cv = cv + + def _score(self, params): + model = self.estimator(**params) + scores = cross_val_score(model, self.X, self.y, cv=self.cv) + return scores.mean() + + +data = load_diabetes() +X, y = data.data, data.target + + +search_config = SearchConfig( + max_depth=list(np.arange(2, 15, 1)), + min_samples_split=list(np.arange(2, 25, 2)), +) + +experiment = SklearnExperiment() +experiment.setup(DecisionTreeRegressor, X, y, cv=4) + +optimizer1 = HillClimbingOptimizer() +optimizer1.add_search(experiment, search_config, n_iter=100, n_jobs=2) +optimizer1.run() diff --git a/examples/v5_API_example/experiments/keras.py b/examples/v5_API_example/experiments/keras.py new file mode 100644 index 00000000..a7cc74fd --- /dev/null +++ b/examples/v5_API_example/experiments/keras.py @@ -0,0 +1,68 @@ +from tensorflow import keras +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split + +from hyperactive.base import BaseExperiment + + +X, y = make_classification(n_samples=1000, n_features=20, random_state=42) +X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2) + + +class KerasMultiLayerPerceptron(BaseExperiment): + """ + A class for creating and evaluating a Keras-based Multi-Layer Perceptron (MLP) model. + + This class inherits from BaseExperiment and is designed to build a simple MLP + using Keras, compile it with the Adam optimizer, and train it on the provided + training data. The model consists of one hidden dense layer with configurable + size and activation function, followed by an output layer with a sigmoid + activation for binary classification. + + Attributes: + X_train (array-like): Training feature data. + X_val (array-like): Validation feature data. + y_train (array-like): Training target data. + y_val (array-like): Validation target data. + + Methods: + _score(**params): Builds, compiles, and trains the MLP model using the + specified parameters for the hidden layer, and returns the validation + accuracy. + """ + + def __init__(self, X_train, X_val, y_train, y_val): + super().__init__() + + self.X_train = X_train + self.X_val = X_val + self.y_train = y_train + self.y_val = y_val + + def _score(self, **params): + dense_layer_0 = params["dense_layer_0"] + activation_layer_0 = params["activation_layer_0"] + + model = keras.Sequential( + [ + keras.layers.Dense( + dense_layer_0, + activation=activation_layer_0, + input_shape=(20,), + ), + keras.layers.Dense(1, activation="sigmoid"), + ] + ) + model.compile( + optimizer=keras.optimizers.Adam(learning_rate=0.01), + loss="binary_crossentropy", + metrics=["accuracy"], + ) + model.fit( + self.X_train, + self.y_train, + batch_size=32, + epochs=10, + validation_data=(self.X_val, self.y_val), + ) + return model.evaluate(X_val, y_val)[1] diff --git a/examples/v5_API_example/experiments/sklearn.py b/examples/v5_API_example/experiments/sklearn.py new file mode 100644 index 00000000..fe231cda --- /dev/null +++ b/examples/v5_API_example/experiments/sklearn.py @@ -0,0 +1,67 @@ +from sklearn.model_selection import cross_val_score +from sklearn.ensemble import GradientBoostingRegressor + + +from hyperactive.base import BaseExperiment + + +class SklearnExperiment(BaseExperiment): + """ + Initializes the SklearnExperiment with the given estimator, data, and cross-validation settings. + + Parameters + ---------- + estimator : object + The machine learning estimator to be used for the experiment. + X : array-like + The input data for training the model. + y : array-like + The target values corresponding to the input data. + cv : int, optional + The number of cross-validation folds (default is 4). + """ + + def __init__(self, estimator, X, y, cv=4): + super().__init__() + + self.estimator = estimator + self.X = X + self.y = y + self.cv = cv + + def _score(self, **params): + model = self.estimator(**params) + scores = cross_val_score(model, self.X, self.y, cv=self.cv) + return scores.mean() + + +class GradientBoostingExperiment(BaseExperiment): + """ + A class for conducting experiments with Gradient Boosting Regressor using cross-validation. + + This class inherits from BaseExperiment and allows users to perform experiments + with the GradientBoostingRegressor from sklearn. Users can specify the input + features, target values, and the number of cross-validation folds. + + Attributes: + estimator (type): The regression model to be used, default is GradientBoostingRegressor. + X (array-like): The input features for the model. + y (array-like): The target values for the model. + cv (int): The number of cross-validation folds. + + Methods: + _score(**params): Evaluates the model using cross-validation and returns the mean score. + """ + + def __init__(self, X, y, cv=4): + super().__init__() + + self.estimator = GradientBoostingRegressor # The user could also predefine the estimator + self.X = X + self.y = y + self.cv = cv + + def _score(self, **params): + model = self.estimator(**params) + scores = cross_val_score(model, self.X, self.y, cv=self.cv) + return scores.mean() diff --git a/examples/v5_API_example/experiments/test_function.py b/examples/v5_API_example/experiments/test_function.py new file mode 100644 index 00000000..22545ddb --- /dev/null +++ b/examples/v5_API_example/experiments/test_function.py @@ -0,0 +1,60 @@ +import numpy as np + +from hyperactive.base import BaseExperiment + + +class SphereFunction(BaseExperiment): + """ + A class representing a Sphere function experiment. + + This class inherits from BaseExperiment and implements a simple + Sphere function, which is a common test function for optimization + algorithms. The function is defined as the sum of the squares of + its input parameters plus a constant. + + Attributes: + const (float): A constant added to the function's result. + n_dim (int): The number of dimensions for the input parameters. + + Methods: + _score(**params): Computes the Sphere function value for the + given parameters. + """ + + def __init__(self, const, n_dim=2): + super().__init__() + + self.const = const + self.n_dim = n_dim + + def _score(self, **params): + return np.sum(np.array(params) ** 2) + self.const + + +class AckleyFunction(BaseExperiment): + """ + A class representing the Ackley function, used as a benchmark for optimization algorithms. + + Attributes: + A (float): A constant used in the calculation of the Ackley function. + + Methods: + _score(**params): Computes the Ackley function value for given parameters 'x0' and 'x1'. + + The Ackley function is a non-convex function used to test optimization algorithms. + """ + + def __init__(self, A): + super().__init__() + self.A = A + + def _score(self, **params): + x = params["x0"] + y = params["x1"] + + loss1 = -self.A * np.exp(-0.2 * np.sqrt(0.5 * (x * x + y * y))) + loss2 = -np.exp(0.5 * (np.cos(2 * np.pi * x) + np.cos(2 * np.pi * y))) + loss3 = np.exp(1) + loss4 = self.A + + return -(loss1 + loss2 + loss3 + loss4) diff --git a/examples/v5_API_example/optimizer.py b/examples/v5_API_example/optimizer.py new file mode 100644 index 00000000..aa354a46 --- /dev/null +++ b/examples/v5_API_example/optimizer.py @@ -0,0 +1,42 @@ +import numpy as np +from sklearn.datasets import load_diabetes +from sklearn.tree import DecisionTreeRegressor + + +from hyperactive.base.search_space_optional import SearchSpace +from hyperactive.optimizers import ( + HillClimbingOptimizer, + RandomRestartHillClimbingOptimizer, +) + +from .experiments.test_function import SklearnExperiment + + +data = load_diabetes() +X, y = data.data, data.target + + +search_space = { + "max_depth": list(np.arange(2, 15, 1)), + "min_samples_split": list(np.arange(2, 25, 2)), +} + +""" optional way of defining search-space +search_space = SearchSpace( + max_depth=list(np.arange(2, 15, 1)), + min_samples_split=list(np.arange(2, 25, 2)), +) +""" + +experiment = SklearnExperiment(DecisionTreeRegressor, X, y, cv=4) + +optimizer1 = HillClimbingOptimizer(n_iter=50) +optimizer2 = RandomRestartHillClimbingOptimizer(n_iter=50, n_jobs=2) + +optimizer1.add_search(experiment, search_space) +optimizer2.add_search(experiment, search_space) + +# not sure about this way of combining optimizers. Might not be intuitive what the plus means. +hyper = optimizer1 + optimizer2 + +hyper.run(max_time=5) diff --git a/pyproject.toml b/pyproject.toml index 89eec432..95843bfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,8 @@ dependencies = [ "numpy >=1.18.1, <3.0.0", "tqdm >=4.48.0, <5.0.0", "pandas <3.0.0", - "gradient-free-optimizers >=1.2.4, <2.0.0", + "gradient-free-optimizers >=1.5.0, <2.0.0", + "scikit-base <1.0.0", ] [project.optional-dependencies] diff --git a/src/hyperactive/__init__.py b/src/hyperactive/__init__.py index bc82b676..18bbdb16 100644 --- a/src/hyperactive/__init__.py +++ b/src/hyperactive/__init__.py @@ -6,11 +6,3 @@ __version__ = importlib.metadata.version("hyperactive") __license__ = "MIT" - - -from .hyperactive import Hyperactive - - -__all__ = [ - "Hyperactive", -] diff --git a/src/hyperactive/experiment/__init__.py b/src/hyperactive/experiment/__init__.py new file mode 100644 index 00000000..deb4ba6b --- /dev/null +++ b/src/hyperactive/experiment/__init__.py @@ -0,0 +1,6 @@ +"""Base classes for optimizers and experiments.""" + +from ._experiment import BaseExperiment +from ._utility import add_callback, add_catch + +__all__ = ["BaseExperiment", "add_callback", "add_catch"] diff --git a/src/hyperactive/experiment/_experiment.py b/src/hyperactive/experiment/_experiment.py new file mode 100644 index 00000000..af0fe2c4 --- /dev/null +++ b/src/hyperactive/experiment/_experiment.py @@ -0,0 +1,91 @@ +"""Base class for experiment.""" + +from abc import ABC, abstractmethod +from typing import Union, List, Dict, Type +from skbase.base import BaseObject + + +class BaseExperiment(ABC, BaseObject): + """Base class for experiment.""" + + def __init__( + self, + catch: Dict = None, + ): + super().__init__() + + self.catch = catch + + self._catch = catch or {} + + def __call__(self, **kwargs): + """Score parameters, with kwargs call.""" + return self.score(kwargs) + + def paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return self._paramnames() + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + raise NotImplementedError + + def score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + paramnames = self.paramnames() + if not set(paramnames) == set(params.keys()): + raise ValueError("Parameters do not match.") + return self._score(**params) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + raise NotImplementedError + + def backend_adapter(self, backend_adapter, s_space): + gfo_wrapper_model = backend_adapter( + experiment=self, + ) + + self.gfo_objective_function = gfo_wrapper_model(s_space()) + + @abstractmethod + def objective_function(self, params): + pass diff --git a/src/hyperactive/experiment/_utility.py b/src/hyperactive/experiment/_utility.py new file mode 100644 index 00000000..f4e38e59 --- /dev/null +++ b/src/hyperactive/experiment/_utility.py @@ -0,0 +1,28 @@ +def add_callback(before=None, after=None): + def decorator(function): + def wrapper(self, param): + if before: + [before_callback(self, param) for before_callback in before] + result = function(self, param) + if after: + [after_callback(self, param) for after_callback in after] + return result + + return wrapper + + return decorator + + +def add_catch(catch): + def decorator(function): + def wrapper(self, param): + try: + result = function(self, param) + except tuple(catch.keys()) as e: + result = catch[e.__class__] + + return result + + return wrapper + + return decorator diff --git a/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py b/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py index ad4e5d5e..a7db6550 100644 --- a/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py +++ b/src/hyperactive/integrations/sklearn/hyperactive_search_cv.py @@ -18,6 +18,7 @@ from .best_estimator import BestEstimator as _BestEstimator_ from .checks import Checks from ...optimizers import RandomSearchOptimizer +from hyperactive.integrations.sklearn.sklearn_cv_experiment import SklearnCvExperiment class HyperactiveSearchCV(BaseEstimator, _BestEstimator_, Checks): @@ -118,12 +119,14 @@ def fit(self, X, y, **fit_params): fit_params = _check_method_params(X, params=fit_params) self.scorer_ = check_scoring(self.estimator, scoring=self.scoring) - objective_function_adapter = ObjectiveFunctionAdapter( - self.estimator, + experiment = SklearnCvExperiment( + estimator=self.estimator, + scoring=self.scorer_, + cv=self.cv, + X=X, + y=y, ) - objective_function_adapter.add_dataset(X, y) - objective_function_adapter.add_validation(self.scorer_, self.cv) - objective_function = objective_function_adapter.objective_function + objective_function = experiment.score hyper = Hyperactive(verbosity=False) hyper.add_search( diff --git a/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py b/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py new file mode 100644 index 00000000..b77aaf7d --- /dev/null +++ b/src/hyperactive/integrations/sklearn/sklearn_cv_experiment.py @@ -0,0 +1,59 @@ +"""Experiment adapter for sklearn cross-validation experiments.""" + +from sklearn import clone +from sklearn.model_selection import cross_validate +from sklearn.utils.validation import _num_samples + +from hyperactive.base import BaseExperiment + +class SklearnCvExperiment(BaseExperiment): + + def __init__(self, estimator, scoring, cv, X, y): + self.estimator = estimator + self.X = X + self.y = y + self.scoring = scoring + self.cv = cv + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return list(self.estimator.get_params().keys()) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + estimator = clone(self.estimator) + estimator.set_params(**params) + + cv_results = cross_validate( + self.estimator, + self.X, + self.y, + cv=self.cv, + ) + + add_info_d = { + "score_time": cv_results["score_time"], + "fit_time": cv_results["fit_time"], + "n_test_samples": _num_samples(self.X), + } + + return cv_results["test_score"].mean(), add_info_d diff --git a/src/hyperactive/integrations/sktime/__init__.py b/src/hyperactive/integrations/sktime/__init__.py new file mode 100644 index 00000000..09bdbd71 --- /dev/null +++ b/src/hyperactive/integrations/sktime/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .main import HyperactiveSearchCV diff --git a/src/hyperactive/integrations/sktime/main.py b/src/hyperactive/integrations/sktime/main.py new file mode 100644 index 00000000..f0d7c5db --- /dev/null +++ b/src/hyperactive/integrations/sktime/main.py @@ -0,0 +1,8 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class HyperactiveSearchCV: + def __init__(self) -> None: + pass diff --git a/tests/_local_test_optimization_strategies/__init__.py b/src/hyperactive/optimization/__init__.py similarity index 100% rename from tests/_local_test_optimization_strategies/__init__.py rename to src/hyperactive/optimization/__init__.py diff --git a/src/hyperactive/optimization/_abstract_optimizer.py b/src/hyperactive/optimization/_abstract_optimizer.py new file mode 100644 index 00000000..eefef594 --- /dev/null +++ b/src/hyperactive/optimization/_abstract_optimizer.py @@ -0,0 +1,16 @@ +from abc import ABC, abstractmethod +from typing import Union, List, Dict, Type +from skbase.base import BaseObject + + +class AbstractOptimizer(ABC, BaseObject): + def __init__(self, *args, **kwargs): + pass + + @abstractmethod + def add_search(self, *args, **kwargs): + pass + + @abstractmethod + def run(self, *args, **kwargs): + pass diff --git a/src/hyperactive/optimization/gradient_free_optimizers/__init__.py b/src/hyperactive/optimization/gradient_free_optimizers/__init__.py new file mode 100644 index 00000000..cd77760f --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/__init__.py @@ -0,0 +1,55 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from ._optimizers import ( + HillClimbingOptimizer, + StochasticHillClimbingOptimizer, + RepulsingHillClimbingOptimizer, + SimulatedAnnealingOptimizer, + DownhillSimplexOptimizer, + RandomSearchOptimizer, + GridSearchOptimizer, + RandomRestartHillClimbingOptimizer, + RandomAnnealingOptimizer, + PowellsMethod, + PatternSearch, + ParallelTemperingOptimizer, + ParticleSwarmOptimizer, + SpiralOptimization, + GeneticAlgorithmOptimizer, + EvolutionStrategyOptimizer, + DifferentialEvolutionOptimizer, + BayesianOptimizer, + LipschitzOptimizer, + DirectAlgorithm, + TreeStructuredParzenEstimators, + ForestOptimizer, +) + + +__all__ = [ + "HillClimbingOptimizer", + "StochasticHillClimbingOptimizer", + "RepulsingHillClimbingOptimizer", + "SimulatedAnnealingOptimizer", + "DownhillSimplexOptimizer", + "RandomSearchOptimizer", + "GridSearchOptimizer", + "RandomRestartHillClimbingOptimizer", + "RandomAnnealingOptimizer", + "PowellsMethod", + "PatternSearch", + "ParallelTemperingOptimizer", + "ParticleSwarmOptimizer", + "SpiralOptimization", + "GeneticAlgorithmOptimizer", + "EvolutionStrategyOptimizer", + "DifferentialEvolutionOptimizer", + "BayesianOptimizer", + "LipschitzOptimizer", + "DirectAlgorithm", + "TreeStructuredParzenEstimators", + "ForestOptimizer", +] diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_base_optimizer.py b/src/hyperactive/optimization/gradient_free_optimizers/_base_optimizer.py new file mode 100644 index 00000000..6b07cd2f --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_base_optimizer.py @@ -0,0 +1,187 @@ +"""Base class for optimizer.""" + +from typing import Union, List, Dict +import multiprocessing as mp +import pandas as pd + +from .search_space import SearchSpace +from ._search import Search + +from ._search_info import SearchInfo + + +from ._composite_optimizer import CompositeOptimizer + +from .._abstract_optimizer import AbstractOptimizer + + +class BaseOptimizer(AbstractOptimizer): + """Base class for optimizer.""" + + n_search: int + searches: list + opt_pros: dict + + def __init__(self, optimizer_class, opt_params): + super().__init__() + + self.optimizer_class = optimizer_class + self.opt_params = opt_params + + self.n_search = 0 + self.searches = [] + + @staticmethod + def _default_search_id(search_id, objective_function): + if not search_id: + search_id = objective_function.__name__ + return search_id + + @staticmethod + def check_list(search_space): + for key in search_space.keys(): + search_dim = search_space[key] + + error_msg = ( + "Value in '{}' of search space dictionary must be of type list".format( + key + ) + ) + if not isinstance(search_dim, list): + print("Warning", error_msg) + # raise ValueError(error_msg) + + def add_search( + self, + experiment: callable, + search_space: Dict[str, list], + n_iter: int, + search_id=None, + n_jobs: int = 1, + initialize: Dict[str, int] = {"grid": 4, "random": 2, "vertices": 4}, + constraints: List[callable] = None, + pass_through: Dict = None, + max_score: float = None, + early_stopping: Dict = None, + random_state: int = None, + memory: Union[str, bool] = "share", + memory_warm_start: pd.DataFrame = None, + ): + """ + Add a new optimization search process with specified parameters. + + Parameters: + - experiment: Experiment class containing the objective-function to optimize. + - search_space: Dictionary defining the search space for optimization. + - n_iter: Number of iterations for the optimization process. + - search_id: Identifier for the search process (default: None). + - n_jobs: Number of parallel jobs to run (default: 1). + - initialize: Dictionary specifying initialization parameters (default: {"grid": 4, "random": 2, "vertices": 4}). + - constraints: List of constraint functions (default: None). + - pass_through: Dictionary of additional parameters to pass through (default: None). + - callbacks: Dictionary of callback functions (default: None). + - catch: Dictionary of exceptions to catch during optimization (default: None). + - max_score: Maximum score to achieve (default: None). + - early_stopping: Dictionary specifying early stopping criteria (default: None). + - random_state: Seed for random number generation (default: None). + - memory: Option to share memory between processes (default: "share"). + - memory_warm_start: DataFrame containing warm start memory (default: None). + """ + + self.n_search += 1 + + self.check_list(search_space) + + constraints = constraints or [] + pass_through = pass_through or {} + early_stopping = early_stopping or {} + + search_id = self._default_search_id(search_id, experiment.objective_function) + s_space = SearchSpace(search_space) + + n_jobs = mp.cpu_count() if n_jobs == -1 else n_jobs + + search_info = SearchInfo( + experiment, + s_space, + n_iter, + initialize, + constraints, + max_score, + early_stopping, + random_state, + memory, + memory_warm_start, + ) + + for _ in range(n_jobs): + search = Search(self.optimizer_class, self.opt_params) + search.setup(search_info) + self.searches.append(search) + + @property + def nth_search(self): + return len(self.composite_opt.optimizers) + + def __add__(self, optimizer_instance): + return CompositeOptimizer(self, optimizer_instance) + + def run( + self, + max_time=None, + distribution: str = "multiprocessing", + n_processes: Union[str, int] = "auto", + verbosity: list = ["progress_bar", "print_results", "print_times"], + ): + + self.comp_opt = CompositeOptimizer(self) + self.comp_opt.run(max_time, distribution, n_processes, verbosity) + + def best_para(self, experiment): + """ + Retrieve the best parameters for a specific ID from the results. + + Parameters: + - experiment (int): The experiment of the optimization run. + + Returns: + - Union[Dict[str, Union[int, float]], None]: The best parameters for the specified ID if found, otherwise None. + + Raises: + - ValueError: If the objective function name is not recognized. + """ + + return self.comp_opt.results_.best_para(experiment.objective_function) + + def best_score(self, experiment): + """ + Return the best score for a specific ID from the results. + + Parameters: + - experiment (int): The experiment of the optimization run. + """ + + return self.comp_opt.results_.best_score(experiment.objective_function) + + def search_data(self, experiment, times=False): + """ + Retrieve search data for a specific ID from the results. Optionally exclude evaluation and iteration times if 'times' is set to False. + + Parameters: + - experiment (int): The experiment of the optimization run. + - times (bool, optional): Whether to exclude evaluation and iteration times. Defaults to False. + + Returns: + - pd.DataFrame: The search data for the specified ID. + """ + + search_data_ = self.comp_opt.results_.search_data(experiment.objective_function) + + if times == False: + search_data_.drop( + labels=["eval_times", "iter_times"], + axis=1, + inplace=True, + errors="ignore", + ) + return search_data_ diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_composite_optimizer.py b/src/hyperactive/optimization/gradient_free_optimizers/_composite_optimizer.py new file mode 100644 index 00000000..bb174557 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_composite_optimizer.py @@ -0,0 +1,60 @@ +from typing import Union +from ._distribution import run_search +from ._results import Results +from ._print_results import PrintResults + +from ._run_info import RunInfo + + +class CompositeOptimizer: + optimizers: list + + def __init__(self, *optimizers): + self.optimizers = list(optimizers) + + def __add__(self, optimizer_instance): + self.optimizers.append(optimizer_instance) + return self + + def run( + self, + max_time=None, + distribution: str = "multiprocessing", + n_processes: Union[str, int] = "auto", + verbosity: list = ["progress_bar", "print_results", "print_times"], + ): + if not verbosity: + verbosity = [] + + run_info = RunInfo( + max_time, + distribution, + n_processes, + verbosity, + ) + + self.collected_searches = [] + for optimizer in self.optimizers: + self.collected_searches += optimizer.searches + + for nth_process, search in enumerate(self.collected_searches): + search.pass_args(max_time, nth_process, verbosity) + + self.results_list = run_search( + self.collected_searches, distribution, n_processes + ) + + self.results_ = Results(self.results_list, self.collected_searches) + + self._print_info(verbosity) + + def _print_info(self, verbosity): + print_res = PrintResults(self.collected_searches, verbosity) + + if verbosity: + for _ in range(len(self.collected_searches)): + print("") + + for results in self.results_list: + nth_process = results["nth_process"] + print_res.print_process(results, nth_process) diff --git a/src/hyperactive/optimizers/constraint.py b/src/hyperactive/optimization/gradient_free_optimizers/_constraint.py similarity index 100% rename from src/hyperactive/optimizers/constraint.py rename to src/hyperactive/optimization/gradient_free_optimizers/_constraint.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_distribution.py b/src/hyperactive/optimization/gradient_free_optimizers/_distribution.py new file mode 100644 index 00000000..6b060f67 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_distribution.py @@ -0,0 +1,86 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from sys import platform +from tqdm import tqdm + +from ._process import _process_ + + +if platform.startswith("linux"): + initializer = tqdm.set_lock + initargs = (tqdm.get_lock(),) +else: + initializer = None + initargs = () + + +def proxy(args): + return _process_(*args) + + +def single_process(process_func, process_infos): + return [process_func(info) for info in process_infos] + + +def multiprocessing_wrapper(process_func, process_infos, n_processes): + import multiprocessing as mp + + process_infos = tuple(process_infos) + + print("\n process_infos ", process_infos) + + with mp.Pool(n_processes, initializer=initializer, initargs=initargs) as pool: + return pool.map(process_func, process_infos) + + +def pathos_wrapper(process_func, search_processes_paras, n_processes): + import pathos.multiprocessing as pmp + + with pmp.Pool(n_processes, initializer=initializer, initargs=initargs) as pool: + return pool.map(process_func, search_processes_paras) + + +def joblib_wrapper(process_func, search_processes_paras, n_processes): + from joblib import Parallel, delayed + + jobs = [delayed(process_func)(*info_dict) for info_dict in search_processes_paras] + return Parallel(n_jobs=n_processes)(jobs) + + +dist_dict = { + "joblib": (joblib_wrapper, _process_), + "multiprocessing": (multiprocessing_wrapper, proxy), + "pathos": (pathos_wrapper, proxy), +} + + +def _get_distribution(distribution): + if hasattr(distribution, "__call__"): + return (distribution, _process_), {} + + elif isinstance(distribution, dict): + dist_key = list(distribution.keys())[0] + dist_paras = list(distribution.values())[0] + + return dist_dict[dist_key], dist_paras + + elif isinstance(distribution, str): + return dist_dict[distribution], {} + + +def run_search(searches, distribution, n_processes): + if n_processes == "auto": + n_processes = len(searches) + + searches_tuple = [(search,) for search in searches] + + if n_processes == 1: + results_list = single_process(_process_, searches) + else: + (distribution, process_func), dist_paras = _get_distribution(distribution) + + results_list = distribution(process_func, searches_tuple, n_processes) + + return results_list diff --git a/src/hyperactive/optimizers/optimizer_attributes.py b/src/hyperactive/optimization/gradient_free_optimizers/_optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimizers/optimizer_attributes.py rename to src/hyperactive/optimization/gradient_free_optimizers/_optimizer_attributes.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py b/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py new file mode 100644 index 00000000..145881b2 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_optimizers.py @@ -0,0 +1,147 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from ._base_optimizer import BaseOptimizer + +from gradient_free_optimizers import ( + HillClimbingOptimizer as _HillClimbingOptimizer, + StochasticHillClimbingOptimizer as _StochasticHillClimbingOptimizer, + RepulsingHillClimbingOptimizer as _RepulsingHillClimbingOptimizer, + SimulatedAnnealingOptimizer as _SimulatedAnnealingOptimizer, + DownhillSimplexOptimizer as _DownhillSimplexOptimizer, + RandomSearchOptimizer as _RandomSearchOptimizer, + GridSearchOptimizer as _GridSearchOptimizer, + RandomRestartHillClimbingOptimizer as _RandomRestartHillClimbingOptimizer, + RandomAnnealingOptimizer as _RandomAnnealingOptimizer, + PowellsMethod as _PowellsMethod, + PatternSearch as _PatternSearch, + ParallelTemperingOptimizer as _ParallelTemperingOptimizer, + ParticleSwarmOptimizer as _ParticleSwarmOptimizer, + SpiralOptimization as _SpiralOptimization_, + GeneticAlgorithmOptimizer as _GeneticAlgorithmOptimizer, + EvolutionStrategyOptimizer as _EvolutionStrategyOptimizer, + DifferentialEvolutionOptimizer as _DifferentialEvolutionOptimizer, + BayesianOptimizer as _BayesianOptimizer, + LipschitzOptimizer as _LipschitzOptimizer_, + DirectAlgorithm as _DirectAlgorithm_, + TreeStructuredParzenEstimators as _TreeStructuredParzenEstimators, + ForestOptimizer as _ForestOptimizer, + EnsembleOptimizer as _EnsembleOptimizer, +) + + +class HillClimbingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_HillClimbingOptimizer, opt_params) + + +class StochasticHillClimbingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_StochasticHillClimbingOptimizer, opt_params) + + +class RepulsingHillClimbingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_RepulsingHillClimbingOptimizer, opt_params) + + +class SimulatedAnnealingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_SimulatedAnnealingOptimizer, opt_params) + + +class DownhillSimplexOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_DownhillSimplexOptimizer, opt_params) + + +class RandomSearchOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_RandomSearchOptimizer, opt_params) + + +class GridSearchOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_GridSearchOptimizer, opt_params) + + +class RandomRestartHillClimbingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_RandomRestartHillClimbingOptimizer, opt_params) + + +class RandomAnnealingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_RandomAnnealingOptimizer, opt_params) + + +class PowellsMethod(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_PowellsMethod, opt_params) + + +class PatternSearch(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_PatternSearch, opt_params) + + +class ParallelTemperingOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_ParallelTemperingOptimizer, opt_params) + + +class ParticleSwarmOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_ParticleSwarmOptimizer, opt_params) + + +class SpiralOptimization(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_SpiralOptimization_, opt_params) + + +class GeneticAlgorithmOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_GeneticAlgorithmOptimizer, opt_params) + + +class EvolutionStrategyOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_EvolutionStrategyOptimizer, opt_params) + + +class DifferentialEvolutionOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_DifferentialEvolutionOptimizer, opt_params) + + +class BayesianOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_BayesianOptimizer, opt_params) + + +class LipschitzOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_LipschitzOptimizer_, opt_params) + + +class DirectAlgorithm(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_DirectAlgorithm_, opt_params) + + +class TreeStructuredParzenEstimators(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_TreeStructuredParzenEstimators, opt_params) + + +class ForestOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_ForestOptimizer, opt_params) + + +class EnsembleOptimizer(BaseOptimizer): + def __init__(self, **opt_params): + super().__init__(_EnsembleOptimizer, opt_params) diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_print_results.py b/src/hyperactive/optimization/gradient_free_optimizers/_print_results.py new file mode 100644 index 00000000..b92dbc81 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_print_results.py @@ -0,0 +1,173 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import logging +import numpy as np + +indent = " " + + +class PrintResults: + def __init__(self, opt_pros, verbosity): + self.opt_pros = opt_pros + self.verbosity = verbosity + + def _print_times(self, eval_time, iter_time, n_iter): + opt_time = iter_time - eval_time + iterPerSec = n_iter / iter_time + + print( + indent, + "Evaluation time :", + eval_time, + "sec", + indent, + "[{} %]".format(round(eval_time / iter_time * 100, 2)), + ) + print( + indent, + "Optimization time :", + opt_time, + "sec", + indent, + "[{} %]".format(round(opt_time / iter_time * 100, 2)), + ) + if iterPerSec >= 1: + print( + indent, + "Iteration time :", + iter_time, + "sec", + indent, + "[{} iter/sec]".format(round(iterPerSec, 2)), + ) + else: + secPerIter = iter_time / n_iter + print( + indent, + "Iteration time :", + iter_time, + "sec", + indent, + "[{} sec/iter]".format(round(secPerIter, 2)), + ) + print(" ") + + def align_para_names(self, para_names): + str_lengths = [len(str_) for str_ in para_names] + max_length = max(str_lengths) + + para_names_align = {} + for para_name, str_length in zip(para_names, str_lengths): + added_spaces = max_length - str_length + para_names_align[para_name] = " " * added_spaces + + return para_names_align + + def _print_results( + self, + experiment, + best_score, + best_para, + best_iter, + best_additional_results, + random_seed, + ): + print("\nResults: '{}'".format(experiment.__class__.__name__), " ") + if best_para is None: + print(indent, "Best score:", best_score, " ") + print(indent, "Best parameter set:", best_para, " ") + print(indent, "Best iteration:", best_iter, " ") + + else: + print(indent, "Best score:", best_score, " ") + + if best_additional_results: + print(indent, "Best additional results:") + add_results_names = list(best_additional_results.keys()) + add_results_names_align = self.align_para_names( + add_results_names + ) + + for best_additional_result in best_additional_results.keys(): + added_spaces = add_results_names_align[ + best_additional_result + ] + print( + indent, + indent, + "'{}'".format(best_additional_result), + "{}:".format(added_spaces), + best_additional_results[best_additional_result], + " ", + ) + + if best_para: + print(indent, "Best parameter set:") + para_names = list(best_para.keys()) + para_names_align = self.align_para_names(para_names) + + for para_key in best_para.keys(): + added_spaces = para_names_align[para_key] + print( + indent, + indent, + "'{}'".format(para_key), + "{}:".format(added_spaces), + best_para[para_key], + " ", + ) + + print(indent, "Best iteration:", best_iter, " ") + + print(" ") + print(indent, "Random seed:", random_seed, " ") + print(" ") + + def print_process(self, results, nth_process): + verbosity = self.verbosity + experiment = self.opt_pros[nth_process].experiment + search_space = self.opt_pros[nth_process].s_space.search_space + + search_data = results["search_data"] + + try: + best_sample = search_data.iloc[search_data["score"].idxmax()] + + except TypeError: + logging.warning( + "Warning: Cannot index by location index with a non-integer key" + ) + + else: + best_score = best_sample["score"] + best_values = best_sample[list(search_space.keys())] + best_para = dict(zip(list(search_space.keys()), best_values)) + best_additional_results_df = best_sample.drop( + ["score"] + list(search_space.keys()) + ) + best_additional_results = best_additional_results_df.to_dict() + + best_iter = results["best_iter"] + eval_times = results["eval_times"] + iter_times = results["iter_times"] + random_seed = results["random_seed"] + + n_iter = self.opt_pros[nth_process].n_iter + + eval_time = np.array(eval_times).sum() + iter_time = np.array(iter_times).sum() + + if "print_results" in verbosity: + self._print_results( + experiment, + best_score, + best_para, + best_iter, + best_additional_results, + random_seed, + ) + + if "print_times" in verbosity: + self._print_times(eval_time, iter_time, n_iter) diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_process.py b/src/hyperactive/optimization/gradient_free_optimizers/_process.py new file mode 100644 index 00000000..3113743f --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_process.py @@ -0,0 +1,36 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from tqdm import tqdm + + +def _process_(optimizer): + if "progress_bar" in optimizer.verbosity: + p_bar = tqdm( + position=optimizer.nth_process, + total=optimizer.n_iter, + ascii=" ─", + colour="Yellow", + ) + else: + p_bar = None + + optimizer._search(p_bar) + + if p_bar: + p_bar.colour = "GREEN" + p_bar.refresh() + p_bar.close() + + return { + "nth_process": optimizer.nth_process, + "best_para": optimizer.best_para, + "best_score": optimizer.best_score, + "best_iter": optimizer.best_since_iter, + "eval_times": optimizer.eval_times, + "iter_times": optimizer.iter_times, + "search_data": optimizer.search_data, + "random_seed": optimizer.random_seed, + } diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_results.py b/src/hyperactive/optimization/gradient_free_optimizers/_results.py new file mode 100644 index 00000000..d92f105f --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_results.py @@ -0,0 +1,88 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +import numpy as np +import pandas as pd + + +class Results: + def __init__(self, results_list, opt_pros): + self.results_list = results_list + self.opt_pros = opt_pros + + self.objFunc2results = {} + self.search_id2results = {} + + def _sort_results_objFunc(self, objective_function): + best_score = -np.inf + best_para = None + search_data = None + + search_data_list = [] + + for results_ in self.results_list: + nth_process = results_["nth_process"] + + opt = self.opt_pros[nth_process] + objective_function_ = opt.experiment.objective_function + search_space_ = opt.s_space() + params = list(search_space_.keys()) + + if objective_function_ != objective_function: + continue + + if results_["best_score"] > best_score: + best_score = results_["best_score"] + best_para = results_["best_para"] + + search_data = results_["search_data"] + search_data["eval_times"] = results_["eval_times"] + search_data["iter_times"] = results_["iter_times"] + + search_data_list.append(search_data) + + if len(search_data_list) > 0: + search_data = pd.concat(search_data_list) + + self.objFunc2results[objective_function] = { + "best_para": best_para, + "best_score": best_score, + "search_data": search_data, + "params": params, + } + + def _get_result(self, id_, result_name): + if id_ not in self.objFunc2results: + self._sort_results_objFunc(id_) + + search_data = self.objFunc2results[id_][result_name] + + return search_data + + def best_para(self, id_): + best_para_ = self._get_result(id_, "best_para") + + if best_para_ is not None: + return best_para_ + + raise ValueError("objective function name not recognized") + + def best_score(self, id_): + best_score_ = self._get_result(id_, "best_score") + + if best_score_ != -np.inf: + return best_score_ + + raise ValueError("objective function name not recognized") + + def search_data(self, id_): + search_data = self._get_result(id_, "search_data") + + params = self.objFunc2results[id_]["params"] + + if search_data is not None: + return search_data + + raise ValueError("objective function name not recognized") diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_run_info.py b/src/hyperactive/optimization/gradient_free_optimizers/_run_info.py new file mode 100644 index 00000000..9bfd6352 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_run_info.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass() +class RunInfo: + max_time: int + distribution: int + n_processes: int + verbosity: int diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_runner.py b/src/hyperactive/optimization/gradient_free_optimizers/_runner.py new file mode 100644 index 00000000..f658ddca --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_runner.py @@ -0,0 +1,100 @@ +from .adapter._hyper_gradient_conv import HyperGradientConv + + +class Runner: + def __init__(self, optimizer_class, opt_params): + self.optimizer_class = optimizer_class + self.opt_params = opt_params + + def run_search(self, search_info, nth_process, max_time, p_bar): + self.search_info = search_info + + self.hg_conv = HyperGradientConv(search_info.s_space) + + search_space_positions = search_info.s_space.positions + initialize = self.hg_conv.conv_initialize(search_info.initialize) + + # conv warm start for smbo from values into positions + if "warm_start_smbo" in self.opt_params: + self.opt_params["warm_start_smbo"] = self.hg_conv.conv_memory_warm_start( + self.opt_params["warm_start_smbo"] + ) + + gfo_constraints = [ + Constraint(constraint, self.s_space) + for constraint in search_info.constraints + ] + + memory_warm_start = self.hg_conv.conv_memory_warm_start( + search_info.memory_warm_start + ) + + self.gfo_optimizer = self.optimizer_class( + search_space=search_space_positions, + initialize=initialize, + constraints=gfo_constraints, + random_state=search_info.random_state, + nth_process=nth_process, + **self.opt_params, + ) + + self.gfo_optimizer.init_search( + search_info.experiment.gfo_objective_function, + search_info.n_iter, + max_time, + search_info.max_score, + search_info.early_stopping, + search_info.memory, + memory_warm_start, + False, + ) + for nth_iter in range(search_info.n_iter): + if p_bar: + p_bar.set_description( + "[" + + str(nth_process) + + "] " + + str(search_info.experiment.__class__.__name__) + + " (" + + self.optimizer_class.name + + ")", + ) + + self.gfo_optimizer.search_step(nth_iter) + if self.gfo_optimizer.stop.check(): + break + + if p_bar: + p_bar.set_postfix( + best_score=str(self.gfo_optimizer.score_best), + best_pos=str(self.gfo_optimizer.pos_best), + best_iter=str(self.gfo_optimizer.p_bar._best_since_iter), + ) + + p_bar.update(1) + p_bar.refresh() + + self.gfo_optimizer.finish_search() + + def convert_results2hyper(self): + self.eval_times = sum(self.gfo_optimizer.eval_times) + self.iter_times = sum(self.gfo_optimizer.iter_times) + + if self.gfo_optimizer.best_para is not None: + value = self.hg_conv.para2value(self.gfo_optimizer.best_para) + position = self.hg_conv.position2value(value) + best_para = self.hg_conv.value2para(position) + self.best_para = best_para + else: + self.best_para = None + + self.best_score = self.gfo_optimizer.best_score + self.positions = self.gfo_optimizer.search_data + self.search_data = self.hg_conv.positions2results(self.positions) + + results_dd = self.gfo_optimizer.search_data.drop_duplicates( + subset=self.search_info.s_space.dim_keys, keep="first" + ) + self.memory_values_df = results_dd[ + self.search_info.s_space.dim_keys + ["score"] + ].reset_index(drop=True) diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_search.py b/src/hyperactive/optimization/gradient_free_optimizers/_search.py new file mode 100644 index 00000000..7cf1621e --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_search.py @@ -0,0 +1,67 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from ._optimizer_attributes import OptimizerAttributes +from ._constraint import Constraint + +from .adapter._hyper_gradient_conv import ( + HyperGradientConv, +) +from .adapter import Adapter +from ._runner import Runner + + +class Search(OptimizerAttributes): + max_time: float + nth_process: int + + def __init__(self, optimizer_class, opt_params): + super().__init__() + self.optimizer_class = optimizer_class + self.opt_params = opt_params + + self.runner = Runner(optimizer_class, opt_params) + + def setup(self, search_info): + self.search_info = search_info + + self.adapter = Adapter(search_info) + + self.experiment = search_info.experiment + self.s_space = search_info.s_space + self.n_iter = search_info.n_iter + + self.initialize = search_info.initialize + self.constraints = search_info.constraints + self.max_score = search_info.max_score + self.early_stopping = search_info.early_stopping + self.random_state = search_info.random_state + self.memory = search_info.memory + self.memory_warm_start = search_info.memory_warm_start + + def pass_args(self, max_time, nth_process, verbosity): + self.max_time = max_time + self.nth_process = nth_process + + if "progress_bar" in verbosity: + self.verbosity = ["progress_bar"] + else: + self.verbosity = [] + + def _search(self, p_bar): + self.experiment.backend_adapter(self.adapter.objective_function, self.s_space) + + self.runner.run_search(self.search_info, self.nth_process, self.max_time, p_bar) + + self.runner.convert_results2hyper() + + self._add_result_attributes( + self.runner.best_para, + self.runner.best_score, + self.runner.gfo_optimizer.p_bar._best_since_iter, + self.runner.eval_times, + self.runner.iter_times, + self.runner.search_data, + self.runner.gfo_optimizer.random_seed, + ) diff --git a/src/hyperactive/optimization/gradient_free_optimizers/_search_info.py b/src/hyperactive/optimization/gradient_free_optimizers/_search_info.py new file mode 100644 index 00000000..23c6b623 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/_search_info.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass() +class SearchInfo: + experiment: int + s_space: int + n_iter: int + initialize: int + constraints: int + max_score: int + early_stopping: int + random_state: int + memory: int + memory_warm_start: int diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py new file mode 100644 index 00000000..ce9326f2 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/__init__.py @@ -0,0 +1,13 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from ._adapter import ( + Adapter, +) + + +__all__ = [ + "Adapter", +] diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py new file mode 100644 index 00000000..d28c4fcf --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_adapter.py @@ -0,0 +1,10 @@ +from ._objective_function import ObjectiveFunction +from ._hyper_gradient_conv import HyperGradientConv +from ._results import Results + + +class Adapter: + def __init__(self, search_info): + self.objective_function = ObjectiveFunction + self.search_space = HyperGradientConv + self.results = Results diff --git a/src/hyperactive/optimizers/dictionary.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_dictionary.py similarity index 100% rename from src/hyperactive/optimizers/dictionary.py rename to src/hyperactive/optimization/gradient_free_optimizers/adapter/_dictionary.py diff --git a/src/hyperactive/optimizers/hyper_gradient_conv.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_hyper_gradient_conv.py similarity index 100% rename from src/hyperactive/optimizers/hyper_gradient_conv.py rename to src/hyperactive/optimization/gradient_free_optimizers/adapter/_hyper_gradient_conv.py diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/_objective_function.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_objective_function.py new file mode 100644 index 00000000..2fa67056 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_objective_function.py @@ -0,0 +1,42 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from ._dictionary import DictClass + + +def gfo2hyper(search_space, para): + values_dict = {} + for _, key in enumerate(search_space.keys()): + pos_ = int(para[key]) + values_dict[key] = search_space[key][pos_] + + return values_dict + + +class ObjectiveFunction(DictClass): + def __init__(self, experiment): + super().__init__() + + self.objective_function = experiment.objective_function + self.catch = experiment._catch + + self.nth_iter = -1 + + def __call__(self, search_space): + # wrapper for GFOs + def _model(para): + self.nth_iter += 1 + para = gfo2hyper(search_space, para) + self.para_dict = para + + try: + results = self.objective_function(self) + except tuple(self.catch.keys()) as e: + results = self.catch[e.__class__] + + return results + + _model.__name__ = self.objective_function.__name__ + return _model diff --git a/src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py new file mode 100644 index 00000000..51be0ab8 --- /dev/null +++ b/src/hyperactive/optimization/gradient_free_optimizers/adapter/_results.py @@ -0,0 +1,3 @@ +class Results: + def __init__(self): + pass diff --git a/src/hyperactive/search_space.py b/src/hyperactive/optimization/gradient_free_optimizers/search_space.py similarity index 100% rename from src/hyperactive/search_space.py rename to src/hyperactive/optimization/gradient_free_optimizers/search_space.py diff --git a/src/hyperactive/optimizers/strategies/__init__.py b/src/hyperactive/optimization/pipelines/__init__.py similarity index 100% rename from src/hyperactive/optimizers/strategies/__init__.py rename to src/hyperactive/optimization/pipelines/__init__.py diff --git a/src/hyperactive/optimizers/strategies/custom_optimization_strategy.py b/src/hyperactive/optimization/pipelines/custom_optimization_strategy.py similarity index 100% rename from src/hyperactive/optimizers/strategies/custom_optimization_strategy.py rename to src/hyperactive/optimization/pipelines/custom_optimization_strategy.py diff --git a/src/hyperactive/optimizers/strategies/optimization_strategy.py b/src/hyperactive/optimization/pipelines/optimization_strategy.py similarity index 100% rename from src/hyperactive/optimizers/strategies/optimization_strategy.py rename to src/hyperactive/optimization/pipelines/optimization_strategy.py diff --git a/src/hyperactive/optimizers/strategies/optimizer_attributes.py b/src/hyperactive/optimization/pipelines/optimizer_attributes.py similarity index 100% rename from src/hyperactive/optimizers/strategies/optimizer_attributes.py rename to src/hyperactive/optimization/pipelines/optimizer_attributes.py diff --git a/src/hyperactive/optimization/talos/__init__.py b/src/hyperactive/optimization/talos/__init__.py new file mode 100644 index 00000000..344704b4 --- /dev/null +++ b/src/hyperactive/optimization/talos/__init__.py @@ -0,0 +1 @@ +from ._optimizers import TalosOptimizer diff --git a/src/hyperactive/optimization/talos/_optimizers.py b/src/hyperactive/optimization/talos/_optimizers.py new file mode 100644 index 00000000..5c98294b --- /dev/null +++ b/src/hyperactive/optimization/talos/_optimizers.py @@ -0,0 +1,12 @@ +from .._abstract_optimizer import AbstractOptimizer + + +class TalosOptimizer(AbstractOptimizer): + def __init__(self): + pass + + def add_search(self): + pass + + def run(self): + pass diff --git a/src/hyperactive/search_config/__init__.py b/src/hyperactive/search_config/__init__.py new file mode 100644 index 00000000..346e9a27 --- /dev/null +++ b/src/hyperactive/search_config/__init__.py @@ -0,0 +1,3 @@ +from ._search_config import SearchConfig + +__all__ = ["SearchConfig"] diff --git a/src/hyperactive/search_config/_properties.py b/src/hyperactive/search_config/_properties.py new file mode 100644 index 00000000..5be0e7c6 --- /dev/null +++ b/src/hyperactive/search_config/_properties.py @@ -0,0 +1,27 @@ +import numpy as np + + +def n_dim(search_space): + return len(search_space) + + +def dim_names(search_space): + return list(search_space.keys()) + + +def position_space(search_space): + return { + key: np.array(range(len(search_space[key]))) + for key in search_space.keys() + } + + +def calculate_properties(func): + def wrapper(self, *args, **kwargs): + func(self, *args, **kwargs) + + self.n_dim = n_dim(self._search_space) + self.dim_names = dim_names(self._search_space) + self.position_space = position_space(self._search_space) + + return wrapper diff --git a/src/hyperactive/search_config/_search_config.py b/src/hyperactive/search_config/_search_config.py new file mode 100644 index 00000000..127d9df0 --- /dev/null +++ b/src/hyperactive/search_config/_search_config.py @@ -0,0 +1,109 @@ +""" +Optional: + +It might make sense to use a class instead of a dictionary for the search-space. +The search-space can have specifc properties, that can be computed from the params (previously called search-space-dictionary). +The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. +""" + +from typing import Union, List, Dict, Type +from collections.abc import MutableMapping + +from ._properties import calculate_properties + + +class SearchSpaceDictLike(MutableMapping): + _search_space: dict = None + + def __init__(self, constraints: List[callable] = None, **param_space): + self._search_space = dict(**param_space) + self.constraints = constraints + + def __getitem__(self, key): + return self._search_space[key] + + def __setitem__(self, key, value): + self._search_space[key] = value + + def __delitem__(self, key): + del self._search_space[key] + + def __iter__(self): + return iter(self._search_space) + + def __len__(self): + return len(self._search_space) + + def __repr__(self): + return f"{self.__class__.__name__}({self._search_space})" + + def __call__(self): + return self._search_space + + def keys(self): + return self._search_space.keys() + + def values(self): + return self._search_space.values() + + def items(self): + return self._search_space.items() + + +class SearchConfig(SearchSpaceDictLike): + + @calculate_properties + def __init__(self, **param_space): + super().__init__(**param_space) + + for key, value in param_space.items(): + setattr(self, key, value) + + @calculate_properties + def __setitem__(self, key, value): + SearchSpaceDictLike.__setitem__(self, key, value) + + @calculate_properties + def __delitem__(self, key): + SearchSpaceDictLike.__delitem__(self, key) + + def print(self, indent=2, max_list_length=5): + """ + Prints the dictionary in a readable format. + + Args: + indent (int): The number of spaces to indent nested structures. + max_list_length (int): The maximum number of items to display from long lists. + """ + + def format_value(value, level=0): + prefix = " " * (level * indent) + if isinstance(value, list): + if len(value) > max_list_length: + # Truncate long lists for readability + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value[:max_list_length] + ) + result += f"{prefix}{' ' * indent}... ({len(value) - max_list_length} more items)\n" + result += f"{prefix}]" + else: + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value + ) + result += f"{prefix}]" + elif isinstance(value, dict): + # Format nested dictionaries + result = "{\n" + for k, v in value.items(): + result += f"{prefix}{' ' * indent}{repr(k)}: {format_value(v, level + 1)},\n" + result += f"{prefix}}}" + else: + result = repr(value) + return result + + for key, value in self.items(): + print(f"{key}: {format_value(value)}") diff --git a/src_old/hyperactive/__init__.py b/src_old/hyperactive/__init__.py new file mode 100644 index 00000000..bc82b676 --- /dev/null +++ b/src_old/hyperactive/__init__.py @@ -0,0 +1,16 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import importlib.metadata + +__version__ = importlib.metadata.version("hyperactive") +__license__ = "MIT" + + +from .hyperactive import Hyperactive + + +__all__ = [ + "Hyperactive", +] diff --git a/src_old/hyperactive/base/__init__.py b/src_old/hyperactive/base/__init__.py new file mode 100644 index 00000000..e2c2df68 --- /dev/null +++ b/src_old/hyperactive/base/__init__.py @@ -0,0 +1,7 @@ +"""Base classes for optimizers and experiments.""" + +from ._experiment import BaseExperiment +from ._optimizer import BaseOptimizer +from .search_space import SearchSpace + +__all__ = ["BaseExperiment", "BaseOptimizer", "SearchSpace"] diff --git a/src_old/hyperactive/base/_experiment.py b/src_old/hyperactive/base/_experiment.py new file mode 100644 index 00000000..ad3a5999 --- /dev/null +++ b/src_old/hyperactive/base/_experiment.py @@ -0,0 +1,71 @@ +"""Base class for experiment.""" + +from skbase.base import BaseObject + + +class BaseExperiment(BaseObject): + """Base class for experiment.""" + + def __init__(self): + super().__init__() + + def __call__(self, **kwargs): + """Score parameters, with kwargs call.""" + return self.score(kwargs) + + def paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return self._paramnames() + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + raise NotImplementedError + + def score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + paramnames = self.paramnames() + if not set(paramnames) == set(params.keys()): + raise ValueError("Parameters do not match.") + return self._score(**params) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + raise NotImplementedError diff --git a/src_old/hyperactive/base/_optimizer.py b/src_old/hyperactive/base/_optimizer.py new file mode 100644 index 00000000..55bf7d1a --- /dev/null +++ b/src_old/hyperactive/base/_optimizer.py @@ -0,0 +1,33 @@ +"""Base class for optimizer.""" + +from skbase.base import BaseObject + + +class BaseOptimizer(BaseObject): + """Base class for optimizer.""" + + def __init__(self): + super().__init__() + + def add_search(self, experiment, search_config: dict): + """Add a new optimization search process with specified parameters. + + Parameters + ---------- + experiment : BaseExperiment + The experiment to optimize parameters for. + search_config : dict with str keys + The search configuration dictionary. + """ + self.experiment = experiment + self.search_config = search_config + + def run(self, max_time=None): + """Run the optimization search process. + + Parameters + ---------- + max_time : float + The maximum time used for the optimization process. + """ + raise NotImplementedError diff --git a/src_old/hyperactive/base/search_space/__init__.py b/src_old/hyperactive/base/search_space/__init__.py new file mode 100644 index 00000000..b341faf0 --- /dev/null +++ b/src_old/hyperactive/base/search_space/__init__.py @@ -0,0 +1,3 @@ +from ._search_space import SearchSpace + +__all__ = ["SearchSpace"] diff --git a/src_old/hyperactive/base/search_space/_properties.py b/src_old/hyperactive/base/search_space/_properties.py new file mode 100644 index 00000000..aa1cf543 --- /dev/null +++ b/src_old/hyperactive/base/search_space/_properties.py @@ -0,0 +1,28 @@ +import numpy as np + + +def n_dim(search_space): + return len(search_space) + + +def dim_names(search_space): + return list(search_space.keys()) + + +def position_space(search_space): + return { + key: np.array(range(len(search_space[key]))) + for key in search_space.keys() + } + + +def calculate_properties(func): + def wrapper(self, *args, **kwargs): + func(self, *args, **kwargs) + + self.n_dim = n_dim(self._search_space) + self.dim_names = dim_names(self._search_space) + self.position_space = position_space(self._search_space) + print(" ---> search-space updated!") + + return wrapper diff --git a/src_old/hyperactive/base/search_space/_search_space.py b/src_old/hyperactive/base/search_space/_search_space.py new file mode 100644 index 00000000..fadb6c6f --- /dev/null +++ b/src_old/hyperactive/base/search_space/_search_space.py @@ -0,0 +1,107 @@ +""" +Optional: + +It might make sense to use a class instead of a dictionary for the search-space. +The search-space can have specifc properties, that can be computed from the params (previously called search-space-dictionary). +The search-space can have a certain size, has n dimensions, some of which are numeric, some of which are categorical. +""" + +from collections.abc import MutableMapping + +from ._properties import calculate_properties + + +class SearchSpaceDictLike(MutableMapping): + _search_space: dict = None + + def __init__(self, **param_space): + self._search_space = dict(**param_space) + + def __getitem__(self, key): + return self._search_space[key] + + def __setitem__(self, key, value): + self._search_space[key] = value + + def __delitem__(self, key): + del self._search_space[key] + + def __iter__(self): + return iter(self._search_space) + + def __len__(self): + return len(self._search_space) + + def __repr__(self): + return f"{self.__class__.__name__}({self._search_space})" + + def __call__(self): + return self._search_space + + def keys(self): + return self._search_space.keys() + + def values(self): + return self._search_space.values() + + def items(self): + return self._search_space.items() + + +class SearchSpace(SearchSpaceDictLike): + + @calculate_properties + def __init__(self, **param_space): + super().__init__(**param_space) + + for key, value in param_space.items(): + setattr(self, key, value) + + @calculate_properties + def __setitem__(self, key, value): + SearchSpaceDictLike.__setitem__(self, key, value) + + @calculate_properties + def __delitem__(self, key): + SearchSpaceDictLike.__delitem__(self, key) + + def print(self, indent=2, max_list_length=5): + """ + Prints the dictionary in a readable format. + + Args: + indent (int): The number of spaces to indent nested structures. + max_list_length (int): The maximum number of items to display from long lists. + """ + + def format_value(value, level=0): + prefix = " " * (level * indent) + if isinstance(value, list): + if len(value) > max_list_length: + # Truncate long lists for readability + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value[:max_list_length] + ) + result += f"{prefix}{' ' * indent}... ({len(value) - max_list_length} more items)\n" + result += f"{prefix}]" + else: + result = "[\n" + result += "".join( + f"{prefix}{' ' * indent}{repr(item)},\n" + for item in value + ) + result += f"{prefix}]" + elif isinstance(value, dict): + # Format nested dictionaries + result = "{\n" + for k, v in value.items(): + result += f"{prefix}{' ' * indent}{repr(k)}: {format_value(v, level + 1)},\n" + result += f"{prefix}}}" + else: + result = repr(value) + return result + + for key, value in self.items(): + print(f"{key}: {format_value(value)}") diff --git a/src/hyperactive/distribution.py b/src_old/hyperactive/distribution.py similarity index 100% rename from src/hyperactive/distribution.py rename to src_old/hyperactive/distribution.py diff --git a/src/hyperactive/hyperactive.py b/src_old/hyperactive/hyperactive.py similarity index 100% rename from src/hyperactive/hyperactive.py rename to src_old/hyperactive/hyperactive.py diff --git a/src_old/hyperactive/integrations/__init__.py b/src_old/hyperactive/integrations/__init__.py new file mode 100644 index 00000000..7d540ced --- /dev/null +++ b/src_old/hyperactive/integrations/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .sklearn import HyperactiveSearchCV diff --git a/src_old/hyperactive/integrations/sklearn/__init__.py b/src_old/hyperactive/integrations/sklearn/__init__.py new file mode 100644 index 00000000..b5e193f9 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .hyperactive_search_cv import HyperactiveSearchCV diff --git a/src_old/hyperactive/integrations/sklearn/best_estimator.py b/src_old/hyperactive/integrations/sklearn/best_estimator.py new file mode 100644 index 00000000..def5f828 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/best_estimator.py @@ -0,0 +1,58 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from sklearn.utils.metaestimators import available_if +from sklearn.utils.deprecation import _deprecate_Xt_in_inverse_transform +from sklearn.exceptions import NotFittedError +from sklearn.utils.validation import check_is_fitted + +from .utils import _estimator_has + + +# NOTE Implementations of following methods from: +# https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/model_selection/_search.py +# Tag: 1.5.1 +class BestEstimator: + + @available_if(_estimator_has("score_samples")) + def score_samples(self, X): + check_is_fitted(self) + return self.best_estimator_.score_samples(X) + + @available_if(_estimator_has("predict")) + def predict(self, X): + check_is_fitted(self) + return self.best_estimator_.predict(X) + + @available_if(_estimator_has("predict_proba")) + def predict_proba(self, X): + check_is_fitted(self) + return self.best_estimator_.predict_proba(X) + + @available_if(_estimator_has("predict_log_proba")) + def predict_log_proba(self, X): + check_is_fitted(self) + return self.best_estimator_.predict_log_proba(X) + + @available_if(_estimator_has("decision_function")) + def decision_function(self, X): + check_is_fitted(self) + return self.best_estimator_.decision_function(X) + + @available_if(_estimator_has("transform")) + def transform(self, X): + check_is_fitted(self) + return self.best_estimator_.transform(X) + + @available_if(_estimator_has("inverse_transform")) + def inverse_transform(self, X=None, Xt=None): + X = _deprecate_Xt_in_inverse_transform(X, Xt) + check_is_fitted(self) + return self.best_estimator_.inverse_transform(X) + + @property + def classes_(self): + _estimator_has("classes_")(self) + return self.best_estimator_.classes_ diff --git a/src_old/hyperactive/integrations/sklearn/checks.py b/src_old/hyperactive/integrations/sklearn/checks.py new file mode 100644 index 00000000..2d107a0a --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/checks.py @@ -0,0 +1,10 @@ +class Checks: + _fit_successful = False + + def verify_fit(function): + def wrapper(self, X, y): + out = function(self, X, y) + self._fit_successful = True + return out + + return wrapper diff --git a/src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py b/src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py new file mode 100644 index 00000000..a7db6550 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/hyperactive_search_cv.py @@ -0,0 +1,172 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from collections.abc import Iterable, Callable +from typing import Union, Dict, Type + +from sklearn.base import BaseEstimator, clone +from sklearn.metrics import check_scoring +from sklearn.utils.validation import indexable, _check_method_params + +from sklearn.base import BaseEstimator as SklearnBaseEstimator +from sklearn.model_selection import BaseCrossValidator + +from hyperactive import Hyperactive + +from .objective_function_adapter import ObjectiveFunctionAdapter +from .best_estimator import BestEstimator as _BestEstimator_ +from .checks import Checks +from ...optimizers import RandomSearchOptimizer +from hyperactive.integrations.sklearn.sklearn_cv_experiment import SklearnCvExperiment + + +class HyperactiveSearchCV(BaseEstimator, _BestEstimator_, Checks): + """ + HyperactiveSearchCV class for hyperparameter tuning using cross-validation with sklearn estimators. + + Parameters: + - estimator: SklearnBaseEstimator + The estimator to be tuned. + - params_config: Dict[str, list] + Dictionary containing the hyperparameter search space. + - optimizer: Union[str, Type[RandomSearchOptimizer]], optional + The optimizer to be used for hyperparameter search, default is "default". + - n_iter: int, optional + Number of parameter settings that are sampled, default is 100. + - scoring: Callable | str | None, optional + Scoring method to evaluate the predictions on the test set. + - n_jobs: int, optional + Number of jobs to run in parallel, default is 1. + - random_state: int | None, optional + Random seed for reproducibility. + - refit: bool, optional + Refit the best estimator with the entire dataset, default is True. + - cv: int | "BaseCrossValidator" | Iterable | None, optional + Determines the cross-validation splitting strategy. + + Methods: + - fit(X, y, **fit_params) + Fit the estimator and tune hyperparameters. + - score(X, y, **params) + Return the score of the best estimator on the input data. + """ + + _required_parameters = ["estimator", "optimizer", "params_config"] + + def __init__( + self, + estimator: "SklearnBaseEstimator", + params_config: Dict[str, list], + optimizer: Union[str, Type[RandomSearchOptimizer]] = "default", + n_iter: int = 100, + *, + scoring: Union[Callable, str, None] = None, + n_jobs: int = 1, + random_state: Union[int, None] = None, + refit: bool = True, + cv=None, + ): + super().__init__() + + self.estimator = estimator + self.params_config = params_config + self.optimizer = optimizer + self.n_iter = n_iter + self.scoring = scoring + self.n_jobs = n_jobs + self.random_state = random_state + self.refit = refit + self.cv = cv + + def _refit(self, X, y=None, **fit_params): + self.best_estimator_ = clone(self.estimator).set_params( + **clone(self.best_params_, safe=False) + ) + + self.best_estimator_.fit(X, y, **fit_params) + return self + + def _check_data(self, X, y): + X, y = indexable(X, y) + if hasattr(self, "_validate_data"): + validate_data = self._validate_data + else: + from sklearn.utils.validation import validate_data + + return validate_data(X, y) + + @Checks.verify_fit + def fit(self, X, y, **fit_params): + """ + Fit the estimator using the provided training data. + + Parameters: + - X: array-like or sparse matrix, shape (n_samples, n_features) + The training input samples. + - y: array-like, shape (n_samples,) or (n_samples, n_outputs) + The target values. + - **fit_params: dict of string -> object + Additional fit parameters. + + Returns: + - self: object + Returns the instance itself. + """ + + X, y = self._check_data(X, y) + + fit_params = _check_method_params(X, params=fit_params) + self.scorer_ = check_scoring(self.estimator, scoring=self.scoring) + + experiment = SklearnCvExperiment( + estimator=self.estimator, + scoring=self.scorer_, + cv=self.cv, + X=X, + y=y, + ) + objective_function = experiment.score + + hyper = Hyperactive(verbosity=False) + hyper.add_search( + objective_function, + search_space=self.params_config, + optimizer=self.optimizer, + n_iter=self.n_iter, + n_jobs=self.n_jobs, + random_state=self.random_state, + ) + hyper.run() + + self.best_params_ = hyper.best_para(objective_function) + self.best_score_ = hyper.best_score(objective_function) + self.search_data_ = hyper.search_data(objective_function) + + if self.refit: + self._refit(X, y, **fit_params) + + return self + + def score(self, X, y=None, **params): + """ + Calculate the score of the best estimator on the input data. + + Parameters: + - X: array-like or sparse matrix of shape (n_samples, n_features) + The input samples. + - y: array-like of shape (n_samples,), default=None + The target values. + - **params: dict + Additional parameters to be passed to the scoring function. + + Returns: + - float + The score of the best estimator on the input data. + """ + + return self.scorer_(self.best_estimator_, X, y, **params) + + @property + def fit_successful(self): + self._fit_successful diff --git a/src_old/hyperactive/integrations/sklearn/objective_function_adapter.py b/src_old/hyperactive/integrations/sklearn/objective_function_adapter.py new file mode 100644 index 00000000..9f8fde4a --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/objective_function_adapter.py @@ -0,0 +1,41 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from sklearn import clone +from sklearn.model_selection import cross_validate +from sklearn.utils.validation import _num_samples + + +class ObjectiveFunctionAdapter: + def __init__(self, estimator) -> None: + self.estimator = estimator + + def add_dataset(self, X, y): + self.X = X + self.y = y + + def add_validation(self, scoring, cv): + self.scoring = scoring + self.cv = cv + + def objective_function(self, params): + + estimator = clone(self.estimator) + estimator.set_params(**params) + + cv_results = cross_validate( + estimator, + self.X, + self.y, + cv=self.cv, + ) + + add_info_d = { + "score_time": cv_results["score_time"], + "fit_time": cv_results["fit_time"], + "n_test_samples": _num_samples(self.X), + } + + return cv_results["test_score"].mean(), add_info_d diff --git a/src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py b/src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py new file mode 100644 index 00000000..b77aaf7d --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/sklearn_cv_experiment.py @@ -0,0 +1,59 @@ +"""Experiment adapter for sklearn cross-validation experiments.""" + +from sklearn import clone +from sklearn.model_selection import cross_validate +from sklearn.utils.validation import _num_samples + +from hyperactive.base import BaseExperiment + +class SklearnCvExperiment(BaseExperiment): + + def __init__(self, estimator, scoring, cv, X, y): + self.estimator = estimator + self.X = X + self.y = y + self.scoring = scoring + self.cv = cv + + def _paramnames(self): + """Return the parameter names of the search. + + Returns + ------- + list of str + The parameter names of the search parameters. + """ + return list(self.estimator.get_params().keys()) + + def _score(self, **params): + """Score the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to score. + + Returns + ------- + float + The score of the parameters. + dict + Additional metadata about the search. + """ + estimator = clone(self.estimator) + estimator.set_params(**params) + + cv_results = cross_validate( + self.estimator, + self.X, + self.y, + cv=self.cv, + ) + + add_info_d = { + "score_time": cv_results["score_time"], + "fit_time": cv_results["fit_time"], + "n_test_samples": _num_samples(self.X), + } + + return cv_results["test_score"].mean(), add_info_d diff --git a/src_old/hyperactive/integrations/sklearn/utils.py b/src_old/hyperactive/integrations/sklearn/utils.py new file mode 100644 index 00000000..6a25cb19 --- /dev/null +++ b/src_old/hyperactive/integrations/sklearn/utils.py @@ -0,0 +1,38 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from sklearn.utils.validation import ( + indexable, + _check_method_params, + check_is_fitted, +) + +# NOTE Implementations of following methods from: +# https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/model_selection/_search.py +# Tag: 1.5.1 + + +def _check_refit(search_cv, attr): + if not search_cv.refit: + raise AttributeError( + f"This {type(search_cv).__name__} instance was initialized with " + f"`refit=False`. {attr} is available only after refitting on the best " + "parameters. You can refit an estimator manually using the " + "`best_params_` attribute" + ) + + +def _estimator_has(attr): + def check(self): + _check_refit(self, attr) + if hasattr(self, "best_estimator_"): + # raise an AttributeError if `attr` does not exist + getattr(self.best_estimator_, attr) + return True + # raise an AttributeError if `attr` does not exist + getattr(self.estimator, attr) + return True + + return check diff --git a/src_old/hyperactive/integrations/sktime/__init__.py b/src_old/hyperactive/integrations/sktime/__init__.py new file mode 100644 index 00000000..09bdbd71 --- /dev/null +++ b/src_old/hyperactive/integrations/sktime/__init__.py @@ -0,0 +1,6 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .main import HyperactiveSearchCV diff --git a/src_old/hyperactive/integrations/sktime/main.py b/src_old/hyperactive/integrations/sktime/main.py new file mode 100644 index 00000000..f0d7c5db --- /dev/null +++ b/src_old/hyperactive/integrations/sktime/main.py @@ -0,0 +1,8 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class HyperactiveSearchCV: + def __init__(self) -> None: + pass diff --git a/src/hyperactive/optimizers/__init__.py b/src_old/hyperactive/optimizers/__init__.py similarity index 100% rename from src/hyperactive/optimizers/__init__.py rename to src_old/hyperactive/optimizers/__init__.py diff --git a/src_old/hyperactive/optimizers/constraint.py b/src_old/hyperactive/optimizers/constraint.py new file mode 100644 index 00000000..e170970e --- /dev/null +++ b/src_old/hyperactive/optimizers/constraint.py @@ -0,0 +1,22 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +def gfo2hyper(search_space, para): + values_dict = {} + for _, key in enumerate(search_space.keys()): + pos_ = int(para[key]) + values_dict[key] = search_space[key][pos_] + + return values_dict + + +class Constraint: + def __init__(self, constraint, search_space): + self.constraint = constraint + self.search_space = search_space + + def __call__(self, para): + para = gfo2hyper(self.search_space, para) + return self.constraint(para) diff --git a/src_old/hyperactive/optimizers/dictionary.py b/src_old/hyperactive/optimizers/dictionary.py new file mode 100644 index 00000000..ca30e652 --- /dev/null +++ b/src_old/hyperactive/optimizers/dictionary.py @@ -0,0 +1,17 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class DictClass: + def __init__(self): + self.para_dict = {} + + def __getitem__(self, key): + return self.para_dict[key] + + def keys(self): + return self.para_dict.keys() + + def values(self): + return self.para_dict.values() diff --git a/src_old/hyperactive/optimizers/hyper_gradient_conv.py b/src_old/hyperactive/optimizers/hyper_gradient_conv.py new file mode 100644 index 00000000..d06d9aa8 --- /dev/null +++ b/src_old/hyperactive/optimizers/hyper_gradient_conv.py @@ -0,0 +1,131 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import numpy as np +import pandas as pd + + +class HyperGradientConv: + def __init__(self, s_space): + self.s_space = s_space + + def value2position(self, value: list) -> list: + return [ + np.abs(v - np.array(space_dim)).argmin() + for v, space_dim in zip(value, self.s_space.values_l) + ] + + def value2para(self, value: list) -> dict: + return {key: p for key, p in zip(self.s_space.dim_keys, value)} + + def para2value(self, para: dict) -> list: + return [para[para_name] for para_name in self.s_space.dim_keys] + + def position2value(self, position): + return [ + space_dim[pos] + for pos, space_dim in zip(position, self.s_space.values_l) + ] + + def para_func2str(self, para): + return { + dim_key: ( + para[dim_key].__name__ + if self.s_space.data_types[dim_key] != "number" + else para[dim_key] + ) + for dim_key in self.s_space.dim_keys + } + + def value_func2str(self, value): + try: + return value.__name__ + except: + return value + + def conv_para(self, para_hyper): + para_gfo = {} + for para in self.s_space.dim_keys: + value_hyper = para_hyper[para] + space_dim = list(self.s_space.func2str[para]) + + if self.s_space.data_types[para] == "number": + value_gfo = np.abs(value_hyper - np.array(space_dim)).argmin() + else: + value_hyper = self.value_func2str(value_hyper) + + if value_hyper in space_dim: + value_gfo = space_dim.index(value_hyper) + else: + raise ValueError( + f"'{value_hyper}' was not found in '{para}'" + ) + + para_gfo[para] = value_gfo + return para_gfo + + def conv_initialize(self, initialize): + if "warm_start" in initialize: + warm_start_l = initialize["warm_start"] + warm_start_gfo = [ + self.conv_para(warm_start) for warm_start in warm_start_l + ] + initialize["warm_start"] = warm_start_gfo + + return initialize + + def get_list_positions(self, list1_values, search_dim): + return [search_dim.index(value2) for value2 in list1_values] + + def values2positions(self, values, search_dim): + return np.array(search_dim).searchsorted(values) + + def positions2results(self, positions): + results_dict = {} + + for para_name in self.s_space.dim_keys: + values_list = self.s_space[para_name] + pos_ = positions[para_name].values + values_ = [values_list[idx] for idx in pos_] + results_dict[para_name] = values_ + + results = pd.DataFrame.from_dict(results_dict) + + diff_list = np.setdiff1d(positions.columns, results.columns) + results[diff_list] = positions[diff_list] + + return results + + def conv_memory_warm_start(self, results): + if results is None: + return results + + results.reset_index(inplace=True, drop=True) + + df_positions_dict = {} + for dim_key in self.s_space.dim_keys: + result_dim_values = list(results[dim_key].values) + search_dim = self.s_space.func2str[dim_key] + + if self.s_space.data_types[dim_key] == "object": + result_dim_values = [ + self.value_func2str(value) for value in result_dim_values + ] + + list1_positions = self.get_list_positions( + result_dim_values, search_dim + ) + else: + list1_positions = self.values2positions( + result_dim_values, search_dim + ) + + df_positions_dict[dim_key] = list1_positions + + results_new = pd.DataFrame(df_positions_dict) + + results_new["score"] = results["score"] + results_new.dropna(how="any", inplace=True) + + return results_new diff --git a/src/hyperactive/optimizers/hyper_optimizer.py b/src_old/hyperactive/optimizers/hyper_optimizer.py similarity index 100% rename from src/hyperactive/optimizers/hyper_optimizer.py rename to src_old/hyperactive/optimizers/hyper_optimizer.py diff --git a/src/hyperactive/optimizers/objective_function.py b/src_old/hyperactive/optimizers/objective_function.py similarity index 100% rename from src/hyperactive/optimizers/objective_function.py rename to src_old/hyperactive/optimizers/objective_function.py diff --git a/src_old/hyperactive/optimizers/optimizer_attributes.py b/src_old/hyperactive/optimizers/optimizer_attributes.py new file mode 100644 index 00000000..4ef67b7b --- /dev/null +++ b/src_old/hyperactive/optimizers/optimizer_attributes.py @@ -0,0 +1,32 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +class OptimizerAttributes: + def __init__(self): + self.best_para = None + self.best_score = None + self.best_since_iter = None + self.eval_times = None + self.iter_times = None + self.search_data = None + self.random_seed = None + + def _add_result_attributes( + self, + best_para, + best_score, + best_since_iter, + eval_times, + iter_times, + search_data, + random_seed, + ): + self.best_para = best_para + self.best_score = best_score + self.best_since_iter = best_since_iter + self.eval_times = eval_times + self.iter_times = iter_times + self.search_data = search_data + self.random_seed = random_seed diff --git a/src/hyperactive/optimizers/optimizers.py b/src_old/hyperactive/optimizers/optimizers.py similarity index 100% rename from src/hyperactive/optimizers/optimizers.py rename to src_old/hyperactive/optimizers/optimizers.py diff --git a/src_old/hyperactive/optimizers/strategies/__init__.py b/src_old/hyperactive/optimizers/strategies/__init__.py new file mode 100644 index 00000000..805bb629 --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/__init__.py @@ -0,0 +1,11 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .custom_optimization_strategy import CustomOptimizationStrategy + + +__all__ = [ + "CustomOptimizationStrategy", +] diff --git a/src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py b/src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py new file mode 100644 index 00000000..ccf7c5f1 --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/custom_optimization_strategy.py @@ -0,0 +1,22 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +from .optimization_strategy import BaseOptimizationStrategy + + +class CustomOptimizationStrategy(BaseOptimizationStrategy): + def __init__(self): + super().__init__() + + self.optimizer_setup_l = [] + self.duration_sum = 0 + + def add_optimizer(self, optimizer, duration=1, early_stopping=None): + self.duration_sum += duration + optimizer_setup = { + "optimizer": optimizer, + "duration": duration, + "early_stopping": early_stopping, + } + self.optimizer_setup_l.append(optimizer_setup) diff --git a/src_old/hyperactive/optimizers/strategies/optimization_strategy.py b/src_old/hyperactive/optimizers/strategies/optimization_strategy.py new file mode 100644 index 00000000..0be4d00e --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/optimization_strategy.py @@ -0,0 +1,127 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + + +from .optimizer_attributes import OptimizerAttributes + + +class BaseOptimizationStrategy(OptimizerAttributes): + def __init__(self): + super().__init__() + + def setup_search( + self, + objective_function, + s_space, + n_iter, + initialize, + constraints, + pass_through, + callbacks, + catch, + max_score, + early_stopping, + random_state, + memory, + memory_warm_start, + verbosity, + ): + self.objective_function = objective_function + self.s_space = s_space + self.n_iter = n_iter + + self.initialize = initialize + self.constraints = constraints + self.pass_through = pass_through + self.callbacks = callbacks + self.catch = catch + self.max_score = max_score + self.early_stopping = early_stopping + self.random_state = random_state + self.memory = memory + self.memory_warm_start = memory_warm_start + self.verbosity = verbosity + + self._max_time = None + + if "progress_bar" in self.verbosity: + self.verbosity = [] + else: + self.verbosity = [] + + @property + def max_time(self): + return self._max_time + + @max_time.setter + def max_time(self, value): + self._max_time = value + + for optimizer_setup in self.optimizer_setup_l: + optimizer_setup["optimizer"].max_time = value + + def search(self, nth_process, p_bar): + for optimizer_setup in self.optimizer_setup_l: + hyper_opt = optimizer_setup["optimizer"] + duration = optimizer_setup["duration"] + opt_strat_early_stopping = optimizer_setup["early_stopping"] + + if opt_strat_early_stopping: + early_stopping = opt_strat_early_stopping + else: + early_stopping = self.early_stopping + + n_iter = round(self.n_iter * duration / self.duration_sum) + + # initialize + if self.best_para is not None: + initialize = {} + if "warm_start" in initialize: + initialize["warm_start"].append(self.best_para) + else: + initialize["warm_start"] = [self.best_para] + else: + initialize = dict(self.initialize) + + # memory_warm_start + if self.search_data is not None: + memory_warm_start = self.search_data + else: + memory_warm_start = self.memory_warm_start + + # warm_start_smbo + if ( + hyper_opt.optimizer_class.optimizer_type == "sequential" + and self.search_data is not None + ): + hyper_opt.opt_params["warm_start_smbo"] = self.search_data + + hyper_opt.setup_search( + objective_function=self.objective_function, + s_space=self.s_space, + n_iter=n_iter, + initialize=initialize, + constraints=self.constraints, + pass_through=self.pass_through, + callbacks=self.callbacks, + catch=self.catch, + max_score=self.max_score, + early_stopping=early_stopping, + random_state=self.random_state, + memory=self.memory, + memory_warm_start=memory_warm_start, + verbosity=self.verbosity, + ) + + hyper_opt.search(nth_process, p_bar) + + self._add_result_attributes( + hyper_opt.best_para, + hyper_opt.best_score, + hyper_opt.best_since_iter, + hyper_opt.eval_times, + hyper_opt.iter_times, + hyper_opt.search_data, + hyper_opt.gfo_optimizer.random_seed, + ) diff --git a/src_old/hyperactive/optimizers/strategies/optimizer_attributes.py b/src_old/hyperactive/optimizers/strategies/optimizer_attributes.py new file mode 100644 index 00000000..894f0c13 --- /dev/null +++ b/src_old/hyperactive/optimizers/strategies/optimizer_attributes.py @@ -0,0 +1,64 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import pandas as pd + + +class OptimizerAttributes: + def __init__(self): + self.best_para = None + self.best_score = None + self.best_since_iter = None + self.eval_times = None + self.iter_times = None + self.search_data = None + self.random_seed = None + + def _add_result_attributes( + self, + best_para, + best_score, + best_since_iter, + eval_times, + iter_times, + search_data, + random_seed, + ): + if self.best_para is None: + self.best_para = best_para + else: + if best_score > self.best_score: + self.best_para = best_para + + if self.best_score is None: + self.best_score = best_score + else: + if best_score > self.best_score: + self.best_score = best_score + + if self.best_since_iter is None: + self.best_since_iter = best_since_iter + else: + if best_score > self.best_score: + self.best_since_iter = best_since_iter + + if self.eval_times is None: + self.eval_times = eval_times + else: + self.eval_times = self.eval_times + eval_times + + if self.iter_times is None: + self.iter_times = iter_times + else: + self.iter_times = self.iter_times + eval_times + + if self.search_data is None: + self.search_data = search_data + else: + self.search_data = pd.concat( + [self.search_data, search_data], ignore_index=True + ) + + if self.random_seed is None: + self.random_seed = random_seed diff --git a/src/hyperactive/print_results.py b/src_old/hyperactive/print_results.py similarity index 100% rename from src/hyperactive/print_results.py rename to src_old/hyperactive/print_results.py diff --git a/src/hyperactive/process.py b/src_old/hyperactive/process.py similarity index 100% rename from src/hyperactive/process.py rename to src_old/hyperactive/process.py diff --git a/src/hyperactive/results.py b/src_old/hyperactive/results.py similarity index 100% rename from src/hyperactive/results.py rename to src_old/hyperactive/results.py diff --git a/src/hyperactive/run_search.py b/src_old/hyperactive/run_search.py similarity index 100% rename from src/hyperactive/run_search.py rename to src_old/hyperactive/run_search.py diff --git a/src_old/hyperactive/search_space.py b/src_old/hyperactive/search_space.py new file mode 100644 index 00000000..101201d8 --- /dev/null +++ b/src_old/hyperactive/search_space.py @@ -0,0 +1,134 @@ +# Author: Simon Blanke +# Email: simon.blanke@yahoo.com +# License: MIT License + +import numpy as np + + +class DictClass: + def __init__(self, search_space): + self.search_space = search_space + + def __getitem__(self, key): + return self.search_space[key] + + def keys(self): + return self.search_space.keys() + + def values(self): + return self.search_space.values() + + +class SearchSpace(DictClass): + def __init__(self, search_space): + super().__init__(search_space) + self.search_space = search_space + + self.dim_keys = list(search_space.keys()) + self.values_l = list(self.search_space.values()) + + positions = {} + for key in search_space.keys(): + positions[key] = np.array(range(len(search_space[key]))) + self.positions = positions + + self.check_list() + self.check_non_num_values() + + self.data_types = self.dim_types() + self.func2str = self._create_num_str_ss() + + def __call__(self): + return self.search_space + + def dim_types(self): + data_types = {} + for dim_key in self.dim_keys: + dim_values = np.array(list(self.search_space[dim_key])) + try: + np.subtract(dim_values, dim_values) + np.array(dim_values).searchsorted(dim_values) + except: + _type_ = "object" + else: + _type_ = "number" + + data_types[dim_key] = _type_ + return data_types + + def _create_num_str_ss(self): + func2str = {} + for dim_key in self.dim_keys: + if self.data_types[dim_key] == "number": + func2str[dim_key] = self.search_space[dim_key] + else: + func2str[dim_key] = [] + + dim_values = self.search_space[dim_key] + for value in dim_values: + try: + func_name = value.__name__ + except: + func_name = value + + func2str[dim_key].append(func_name) + return func2str + + def check_list(self): + for dim_key in self.dim_keys: + search_dim = self.search_space[dim_key] + + err_msg = "\n Value in '{}' of search space dictionary must be of type list \n".format( + dim_key + ) + if not isinstance(search_dim, list): + raise ValueError(err_msg) + + @staticmethod + def is_function(value): + try: + value.__name__ + except: + return False + else: + return True + + @staticmethod + def is_number(value): + try: + float(value) + value * 0.1 + value - 0.1 + value / 0.1 + except: + return False + else: + return True + + def _string_or_object(self, dim_key, dim_values): + for dim_value in dim_values: + is_str = isinstance(dim_value, str) + is_func = self.is_function(dim_value) + is_number = self.is_number(dim_value) + + if not is_str and not is_func and not is_number: + msg = "\n The value '{}' of type '{}' in the search space dimension '{}' must be number, string or function \n".format( + dim_value, type(dim_value), dim_key + ) + raise ValueError(msg) + + def check_non_num_values(self): + for dim_key in self.dim_keys: + dim_values = np.array(list(self.search_space[dim_key])) + + try: + np.subtract(dim_values, dim_values) + np.array(dim_values).searchsorted(dim_values) + except: + self._string_or_object(dim_key, dim_values) + else: + if dim_values.ndim != 1: + msg = "Array-like object in '{}' must be one dimensional".format( + dim_key + ) + raise ValueError(msg) diff --git a/tests/_local_test_timings/__init__.py b/tests/test_api/__init__.py similarity index 100% rename from tests/_local_test_timings/__init__.py rename to tests/test_api/__init__.py diff --git a/tests/test_api/test_callbacks.py b/tests/test_api/test_callbacks.py new file mode 100644 index 00000000..cdac7d73 --- /dev/null +++ b/tests/test_api/test_callbacks.py @@ -0,0 +1,123 @@ +import copy +import pytest +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment, add_callback +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x0=list(np.arange(-10, 10, 1)), +) + +n_iter = 20 + + +def test_callback_0(): + class Experiment(BaseExperiment): + def callback_1(self, access): + access.stuff1 = 1 + + def callback_2(self, access): + access.stuff2 = 2 + + @add_callback(before=[callback_1, callback_2]) + def objective_function(self, access): + assert access.stuff1 == 1 + assert access.stuff2 == 2 + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + ) + hyper.run() + + +def test_callback_1(): + class Experiment(BaseExperiment): + def callback_1(self, access): + access.stuff1 = 1 + + def callback_2(self, access): + access.stuff1 = 2 + + @add_callback(before=[callback_1], after=[callback_2]) + def objective_function(self, access): + assert access.stuff1 == 1 + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + ) + hyper.run() + + +def test_callback_2(): + class Experiment(BaseExperiment): + + def callback_1(self, access): + self.test_var = 1 + + def setup(self, test_var): + self.test_var = test_var + + @add_callback(before=[callback_1]) + def objective_function(self, access): + assert self.test_var == 1 + + return 0 + + experiment = Experiment() + experiment.setup(5) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + ) + hyper.run() + + +def test_callback_3(): + class Experiment(BaseExperiment): + + def callback_1(self, access): + self.test_var = 1 + + def setup(self, test_var): + self.test_var = test_var + + @add_callback(after=[callback_1]) + def objective_function(self, access): + if access.nth_iter == 0: + assert self.test_var == 0 + else: + assert self.test_var == 1 + + return 0 + + experiment = Experiment() + experiment.setup(0) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + ) + hyper.run() diff --git a/tests/test_api/test_catch.py b/tests/test_api/test_catch.py new file mode 100644 index 00000000..29a05858 --- /dev/null +++ b/tests/test_api/test_catch.py @@ -0,0 +1,137 @@ +import copy +import pytest +import math +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment, add_catch +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x0=list(np.arange(-10, 10, 1)), +) + + +def test_catch_1(): + class Experiment(BaseExperiment): + @add_catch({TypeError: np.nan}) + def objective_function(self, access): + a = 1 + "str" + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + +def test_catch_2(): + class Experiment(BaseExperiment): + @add_catch({ValueError: np.nan}) + def objective_function(self, access): + math.sqrt(-10) + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + +def test_catch_3(): + class Experiment(BaseExperiment): + @add_catch({ZeroDivisionError: np.nan}) + def objective_function(self, access): + x = 1 / 0 + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + +def test_catch_all_0(): + class Experiment(BaseExperiment): + @add_catch( + { + TypeError: np.nan, + ValueError: np.nan, + ZeroDivisionError: np.nan, + } + ) + def objective_function(self, access): + a = 1 + "str" + math.sqrt(-10) + x = 1 / 0 + + return 0 + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + nan_ = hyper.search_data(experiment)["score"].values[0] + + assert math.isnan(nan_) + + +def test_catch_all_1(): + catch_return = (np.nan, {"error": True}) + + class Experiment(BaseExperiment): + @add_catch( + { + TypeError: catch_return, + ValueError: catch_return, + ZeroDivisionError: catch_return, + } + ) + def objective_function(self, access): + a = 1 + "str" + math.sqrt(-10) + x = 1 / 0 + + return 0, {"error": False} + + experiment = Experiment() + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100, + ) + hyper.run() + + nan_ = hyper.search_data(experiment)["score"].values[0] + error_ = hyper.search_data(experiment)["error"].values[0] + + assert math.isnan(nan_) + assert error_ == True diff --git a/tests/test_api/test_constr_opt.py b/tests/test_api/test_constr_opt.py new file mode 100644 index 00000000..29890933 --- /dev/null +++ b/tests/test_api/test_constr_opt.py @@ -0,0 +1,154 @@ +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +def test_constr_opt_0(): + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + experiment = Experiment() + + search_config = SearchConfig( + x1=list(np.arange(-15, 15, 1)), + ) + + def constraint_1(para): + print(" para", para) + + return para["x1"] > -5 + + constraints_list = [constraint_1] + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=50, + constraints=constraints_list, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + x0_values = search_data["x1"].values + + print("\n search_data \n", search_data, "\n") + + assert np.all(x0_values > -5) + + +def test_constr_opt_1(): + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -(para["x1"] * para["x1"] + para["x2"] * para["x2"]) + return score + + experiment = Experiment() + + search_config = SearchConfig( + x1=list(np.arange(-10, 10, 1)), + x2=list(np.arange(-10, 10, 1)), + ) + + def constraint_1(para): + return para["x1"] > -5 + + constraints_list = [constraint_1] + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=50, + constraints=constraints_list, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + x0_values = search_data["x1"].values + + print("\n search_data \n", search_data, "\n") + + assert np.all(x0_values > -5) + + +""" +def test_constr_opt_2(): + n_iter = 50 + + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + experiment = Experiment() + + search_config = SearchConfig( + x1=list(np.arange(-10, 10, 0.1)), + ) + + def constraint_1(para): + return para["x1"] > -5 + + def constraint_2(para): + return para["x1"] < 5 + + constraints_list = [constraint_1, constraint_2] + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=50, + constraints=constraints_list, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + x0_values = search_data["x1"].values + + print("\n search_data \n", search_data, "\n") + + assert np.all(x0_values > -5) + assert np.all(x0_values < 5) + + n_new_positions = 0 + n_new_scores = 0 + + n_current_positions = 0 + n_current_scores = 0 + + n_best_positions = 0 + n_best_scores = 0 + + for hyper_optimizer in hyper.opt_pros.values(): + optimizer = hyper_optimizer.gfo_optimizer + + n_new_positions = n_new_positions + len(optimizer.pos_new_list) + n_new_scores = n_new_scores + len(optimizer.score_new_list) + + n_current_positions = n_current_positions + len(optimizer.pos_current_list) + n_current_scores = n_current_scores + len(optimizer.score_current_list) + + n_best_positions = n_best_positions + len(optimizer.pos_best_list) + n_best_scores = n_best_scores + len(optimizer.score_best_list) + + print("\n optimizer", optimizer) + print(" n_new_positions", optimizer.pos_new_list) + print(" n_new_scores", optimizer.score_new_list) + + assert n_new_positions == n_iter + assert n_new_scores == n_iter + + assert n_current_positions == n_current_scores + assert n_current_positions <= n_new_positions + + assert n_best_positions == n_best_scores + assert n_best_positions <= n_new_positions + + assert n_new_positions == n_new_scores +""" diff --git a/tests/test_api/test_distribution.py b/tests/test_api/test_distribution.py new file mode 100644 index 00000000..4e39228e --- /dev/null +++ b/tests/test_api/test_distribution.py @@ -0,0 +1,104 @@ +import numpy as np +from tqdm import tqdm + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + +n_iter = 15 + + +def test_n_jobs_0(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs + + +def test_n_jobs_1(): + n_jobs = 4 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=n_jobs) + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs + + +def test_n_jobs_2(): + n_jobs = 8 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs + + +def test_n_jobs_5(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs * 2 + + +def test_n_jobs_6(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + hyper.add_search(experiment, search_config, n_iter=n_iter, n_jobs=n_jobs) + + hyper.run() + + assert len(hyper.search_data(experiment)) == n_iter * n_jobs * 4 + + +def test_n_jobs_7(): + n_jobs = -1 + + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=n_jobs) + hyper.run() + + +def test_multiprocessing_0(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=2) + hyper.run(distribution="multiprocessing") + + +def test_multiprocessing_1(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15, n_jobs=2) + hyper.run( + distribution={ + "multiprocessing": { + "initializer": tqdm.set_lock, + "initargs": (tqdm.get_lock(),), + } + }, + ) diff --git a/tests/test_api/test_early_stop.py b/tests/test_api/test_early_stop.py new file mode 100644 index 00000000..fbc4c8a4 --- /dev/null +++ b/tests/test_api/test_early_stop.py @@ -0,0 +1,340 @@ +import time +import pytest +import numpy as np + +from hyperactive.optimizers import RandomSearchOptimizer, HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x0"] * para["x0"] + return score + + +experiment = Experiment() +search_space = { + "x0": list(np.arange(0, 100000, 0.1)), +} +search_config = SearchConfig( + x0=list(np.arange(0, 100000, 0.1)), +) + + +def test_early_stop_0(): + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": 0.1, + "tol_rel": 0.1, + } + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + +def test_early_stop_1(): + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": None, + "tol_rel": 5, + } + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + +def test_early_stop_3(): + def objective_function(para): + score = -para["x0"] * para["x0"] + return score + + search_space = { + "x0": list(np.arange(0, 100, 0.1)), + } + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": n_iter_no_change, + } + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100000, + initialize={"warm_start": [{"x0": 0}]}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == (n_iter_no_change + 1) + + +def test_early_stop_4(): + class Experiment(BaseExperiment): + def objective_function(self, para): + return para["x0"] + + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": 1, + "tol_rel": None, + } + + start1 = {"x0": 0} + start2 = {"x0": 1} + start3 = {"x0": 2} + start4 = {"x0": 3} + start5 = {"x0": 4} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + start4, + start4, + start4, + start5, + start5, + start5, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == n_iter + + +def test_early_stop_5(): + class Experiment(BaseExperiment): + def objective_function(self, para): + return para["x0"] + + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": n_iter_no_change, + "tol_abs": 10, + "tol_rel": None, + } + + start1 = {"x0": 0} + start2 = {"x0": 9} + start3 = {"x0": 20} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == (n_iter_no_change + 1) + + +def test_early_stop_6(): + class Experiment(BaseExperiment): + def objective_function(self, para): + return para["x0"] + + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": 5, + "tol_abs": None, + "tol_rel": 10, + } + + start1 = {"x0": 1} + start2 = {"x0": 1.1} + start3 = {"x0": 1.22} + start4 = {"x0": 1.35} + start5 = {"x0": 1.48} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + start4, + start4, + start4, + start5, + start5, + start5, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == n_iter + + +def test_early_stop_7(): + def objective_function(para): + return para["x0"] + + experiment = Experiment() + + search_config = SearchConfig( + x0=list(np.arange(0, 100, 0.01)), + ) + + n_iter_no_change = 5 + early_stopping = { + "n_iter_no_change": n_iter_no_change, + "tol_abs": None, + "tol_rel": 10, + } + + start1 = {"x0": 1} + start2 = {"x0": 1.09} + start3 = {"x0": 1.20} + + warm_start_l = [ + start1, + start1, + start1, + start1, + start1, + start2, + start2, + start2, + start3, + start3, + start3, + ] + n_iter = len(warm_start_l) + + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=n_iter, + initialize={"warm_start": warm_start_l}, + early_stopping=early_stopping, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + n_performed_iter = len(search_data) + + print("\n n_performed_iter \n", n_performed_iter) + print("\n n_iter_no_change \n", n_iter_no_change) + + assert n_performed_iter == (n_iter_no_change + 1) diff --git a/tests/integrations/__init__.py b/tests/test_api/test_empty_output/__init__.py similarity index 100% rename from tests/integrations/__init__.py rename to tests/test_api/test_empty_output/__init__.py diff --git a/tests/test_api/test_empty_output/non_verbose.py b/tests/test_api/test_empty_output/non_verbose.py new file mode 100644 index 00000000..8b313e50 --- /dev/null +++ b/tests/test_api/test_empty_output/non_verbose.py @@ -0,0 +1,23 @@ +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + + +hyper = HillClimbingOptimizer() +hyper.add_search(experiment, search_config, n_iter=30, memory=True) +hyper.run(verbosity=False) diff --git a/tests/test_api/test_empty_output/test_empty_output.py b/tests/test_api/test_empty_output/test_empty_output.py new file mode 100644 index 00000000..cd3a3e33 --- /dev/null +++ b/tests/test_api/test_empty_output/test_empty_output.py @@ -0,0 +1,46 @@ +import os, sys, subprocess + +here = os.path.dirname(os.path.abspath(__file__)) + +verbose_file = os.path.join(here, "verbose.py") +non_verbose_file = os.path.join(here, "non_verbose.py") + + +def _run_subprocess(script): + output = [] + process = subprocess.Popen( + [sys.executable, "-u", script], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # Line buffered + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + ) + # Read output line by line + while True: + line = process.stdout.readline() + if line: + output.append(line) + if not line and process.poll() is not None: + break + + return "".join(output), process.stderr.read() + + +def test_empty_output(): + stdout_non_verb, stderr_non_verb = _run_subprocess(non_verbose_file) + + print("\n stdout_non_verb \n", stdout_non_verb, "\n") + print("\n stderr_non_verb \n", stderr_non_verb, "\n") + + assert not stdout_non_verb + assert not stderr_non_verb + + +def test_output(): + stdout_verb, stderr_verb = _run_subprocess(verbose_file) + + print("\n stdout_verb \n", stdout_verb, "\n") + print("\n stderr_verb \n", stderr_verb, "\n") + + assert "Results:" in stdout_verb diff --git a/tests/test_api/test_empty_output/verbose.py b/tests/test_api/test_empty_output/verbose.py new file mode 100644 index 00000000..a7e16a73 --- /dev/null +++ b/tests/test_api/test_empty_output/verbose.py @@ -0,0 +1,26 @@ +import sys +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + + +hyper = HillClimbingOptimizer() +hyper.add_search(experiment, search_config, n_iter=30, memory=True) +hyper.run() + +sys.stdout.flush() diff --git a/tests/test_api/test_initializers.py b/tests/test_api/test_initializers.py new file mode 100644 index 00000000..a798fe41 --- /dev/null +++ b/tests/test_api/test_initializers.py @@ -0,0 +1,128 @@ +import numpy as np +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + + +search_config = SearchConfig( + x1=list(np.arange(-100, 101, 1)), +) + + +def test_initialize_warm_start_0(): + init = { + "x1": 0, + } + + initialize = {"warm_start": [init]} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) < 0.001 + + +def test_initialize_warm_start_1(): + search_space = { + "x1": list(np.arange(-10, 10, 1)), + } + init = { + "x1": -10, + } + + initialize = {"warm_start": [init]} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert hyper.best_para(experiment) == init + + +def test_initialize_vertices(): + initialize = {"vertices": 2} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=2, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) - 10000 < 0.001 + + +def test_initialize_grid_0(): + search_space = { + "x1": list(np.arange(-1, 2, 1)), + } + initialize = {"grid": 1} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) < 0.001 + + +def test_initialize_grid_1(): + search_space = { + "x1": list(np.arange(-2, 3, 1)), + } + + initialize = {"grid": 1} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize=initialize, + ) + hyper.run() + + assert abs(hyper.best_score(experiment)) - 1 < 0.001 + + +def test_initialize_all_0(): + search_space = { + "x1": list(np.arange(-2, 3, 1)), + } + + initialize = {"grid": 100, "vertices": 100, "random": 100} + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=300, + initialize=initialize, + ) + hyper.run() diff --git a/tests/test_api/test_max_score.py b/tests/test_api/test_max_score.py new file mode 100644 index 00000000..0d46696a --- /dev/null +++ b/tests/test_api/test_max_score.py @@ -0,0 +1,77 @@ +import time +import numpy as np +from sklearn.datasets import load_breast_cancer +from sklearn.model_selection import cross_val_score +from sklearn.tree import DecisionTreeClassifier + +from hyperactive.optimizers import HillClimbingOptimizer, RandomSearchOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x1=list(np.arange(0, 100, 0.1)), +) + + +def test_max_score_0(): + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + experiment = Experiment() + + max_score = -9999 + + hyper = HillClimbingOptimizer( + epsilon=0.01, + rand_rest_p=0, + ) + hyper.add_search( + experiment, + search_config, + n_iter=100000, + initialize={"warm_start": [{"x1": 99}]}, + max_score=max_score, + ) + hyper.run() + + print("\n Results head \n", hyper.search_data(experiment).head()) + print("\n Results tail \n", hyper.search_data(experiment).tail()) + + print("\nN iter:", len(hyper.search_data(experiment))) + + assert -100 > hyper.best_score(experiment) > max_score + + +def test_max_score_1(): + + class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + time.sleep(0.01) + return score + + experiment = Experiment() + + max_score = -9999 + + c_time = time.perf_counter() + hyper = RandomSearchOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=100000, + initialize={"warm_start": [{"x1": 99}]}, + max_score=max_score, + ) + hyper.run() + diff_time = time.perf_counter() - c_time + + print("\n Results head \n", hyper.search_data(experiment).head()) + print("\n Results tail \n", hyper.search_data(experiment).tail()) + + print("\nN iter:", len(hyper.search_data(experiment))) + + assert diff_time < 1 diff --git a/tests/test_api/test_max_time.py b/tests/test_api/test_max_time.py new file mode 100644 index 00000000..a1d4f00f --- /dev/null +++ b/tests/test_api/test_max_time.py @@ -0,0 +1,38 @@ +import time +import numpy as np +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, para): + score = -para["x1"] * para["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(0, 100000, 1)), +) + + +def test_max_time_0(): + c_time1 = time.perf_counter() + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=1000000) + hyper.run(max_time=0.1) + diff_time1 = time.perf_counter() - c_time1 + + assert diff_time1 < 1 + + +def test_max_time_1(): + c_time1 = time.perf_counter() + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=1000000) + hyper.run(max_time=1) + diff_time1 = time.perf_counter() - c_time1 + + assert 0.3 < diff_time1 < 2 diff --git a/tests/test_api/test_random_state.py b/tests/test_api/test_random_state.py new file mode 100644 index 00000000..7b79258c --- /dev/null +++ b/tests/test_api/test_random_state.py @@ -0,0 +1,201 @@ +import numpy as np + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -(opt["x1"] * opt["x1"] + opt["x2"] * opt["x2"]) + return score + + +experiment = Experiment() + + +search_config = SearchConfig( + x1=list(np.arange(-1000, 1000, 0.1)), + x2=list(np.arange(-1000, 1000, 0.1)), +) + + +err = 0.001 + + +def test_random_state_n_jobs_0(): + n_jobs = 2 + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=5, + initialize={"random": 1}, + random_state=1, + n_jobs=n_jobs, + ) + hyper.run() + + results = hyper.search_data(experiment) + + no_dup = results.drop_duplicates(subset=list(search_config.keys())) + print("no_dup", no_dup) + print("results", results) + + print(int(len(results) / n_jobs)) + print(len(no_dup)) + + assert int(len(results) / n_jobs) != len(no_dup) + + +def test_random_state_n_jobs_1(): + n_jobs = 3 + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=5, + initialize={"random": 1}, + random_state=1, + n_jobs=n_jobs, + ) + hyper.run() + + results = hyper.search_data(experiment) + + no_dup = results.drop_duplicates(subset=list(search_config.keys())) + print("no_dup", no_dup) + print("results", results) + + assert int(len(results) / n_jobs) != len(no_dup) + + +def test_random_state_n_jobs_2(): + n_jobs = 4 + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=5, + initialize={"random": 1}, + random_state=1, + n_jobs=n_jobs, + ) + hyper.run() + + results = hyper.search_data(experiment) + + no_dup = results.drop_duplicates(subset=list(search_config.keys())) + print("no_dup", no_dup) + print("results", results) + + assert int(len(results) / n_jobs) != len(no_dup) + + +def test_random_state_0(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=1, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=1, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) < err + + +def test_random_state_1(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=10, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=10, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) < err + + +def test_random_state_2(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=1, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + random_state=10, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) > err + + +def test_no_random_state_0(): + hyper0 = HillClimbingOptimizer() + hyper0.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + ) + hyper0.run() + + hyper1 = HillClimbingOptimizer() + hyper1.add_search( + experiment, + search_config, + n_iter=10, + initialize={"random": 1}, + ) + hyper1.run() + + best_score0 = hyper0.best_score(experiment) + best_score1 = hyper1.best_score(experiment) + + assert abs(best_score0 - best_score1) > err diff --git a/tests/test_api/test_results.py b/tests/test_api/test_results.py new file mode 100644 index 00000000..c90a419f --- /dev/null +++ b/tests/test_api/test_results.py @@ -0,0 +1,226 @@ +import pytest +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + +search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), +) + + +def test_attributes_results_0(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + + +def test_attributes_results_1(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert set(search_config.keys()) < set(hyper.search_data(experiment).columns) + + +def test_attributes_results_2(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert "x1" in list(hyper.search_data(experiment).columns) + + +def test_attributes_results_3(): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=100) + hyper.run() + + assert "score" in list(hyper.search_data(experiment).columns) + + +def test_attributes_results_4(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize={"warm_start": [{"x1": 0}]}, + ) + hyper.run() + + assert 0 in list(hyper.search_data(experiment)["x1"].values) + + +def test_attributes_results_5(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=1, + initialize={"warm_start": [{"x1": 10}]}, + ) + hyper.run() + + print( + "\n x1_results \n", + list(hyper.search_data(experiment)["x1"].values), + ) + + assert 10 in list(hyper.search_data(experiment)["x1"].values) + + +def test_attributes_results_6(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + initialize={"random": 1}, + memory=False, + ) + hyper.run() + + x1_results = list(hyper.search_data(experiment)["x1"].values) + + print("\n x1_results \n", x1_results) + + assert len(set(x1_results)) < len(x1_results) + + +def test_attributes_results_7(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + with pytest.raises(Exception) as e_info: + search_data["eval_times"] + + +def test_attributes_results_8(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + ) + hyper.run() + + search_data = hyper.search_data(experiment) + with pytest.raises(Exception) as e_info: + search_data["iter_times"] + + +def test_attributes_results_9(): + def objective_function(opt): + score = -opt["x1"] * opt["x1"] + return score + + search_space = { + "x1": list(np.arange(0, 10, 1)), + } + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=20, + ) + hyper.run() + + search_data = hyper.search_data(experiment, times=True) + search_data["iter_times"] + search_data["eval_times"] + + +""" +def test_attributes_results_7(): + def objective_function(para): + score = -para["x1"] * para["x1"] + return score + + search_space = { + "x1": np.arange(0, 10, 1), + } + + opt = RandomSearchOptimizer(search_space) + opt.search( + experiment, n_iter=20, initialize={"random": 1}, memory=True + ) + + x1_results = list(opt.results["x1"].values) + + print("\n x1_results \n", x1_results) + + assert len(set(x1_results)) == len(x1_results) + + +def test_attributes_results_8(): + def objective_function(para): + score = -para["x1"] * para["x1"] + return score + + search_space = { + "x1": np.arange(-10, 11, 1), + } + + results = pd.DataFrame(np.arange(-10, 10, 1), columns=["x1"]) + results["score"] = 0 + + opt = RandomSearchOptimizer(search_space) + opt.search( + experiment, + n_iter=100, + initialize={}, + memory=True, + memory_warm_start=results, + ) + + print("\n opt.results \n", opt.results) + + x1_results = list(opt.results["x1"].values) + + assert 10 == x1_results[0] +""" diff --git a/tests/test_api/test_results_methods.py b/tests/test_api/test_results_methods.py new file mode 100644 index 00000000..90f52127 --- /dev/null +++ b/tests/test_api/test_results_methods.py @@ -0,0 +1,245 @@ +import pytest +import numbers +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +class Experiment1(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() +experiment1 = Experiment1() + + +search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), +) + + +def test_attributes_best_score_objective_function_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) + + +def test_attributes_best_score_objective_function_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) + + +""" +def test_attributes_best_score_search_id_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) + + +def test_attributes_best_score_search_id_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + search_id="2", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_score(experiment), numbers.Number) +""" + + +def test_attributes_best_para_objective_function_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para(experiment), dict) + + +def test_attributes_best_para_objective_function_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para(experiment), dict) + + +""" +def test_attributes_best_para_search_id_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para("1"), dict) + + +def test_attributes_best_para_search_id_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + search_id="2", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.best_para("1"), dict) +""" + + +def test_attributes_results_objective_function_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + + +def test_attributes_results_objective_function_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + + +""" +def test_attributes_results_search_id_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data("1"), pd.DataFrame) + + +def test_attributes_results_search_id_1(): + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + search_id="1", + n_iter=15, + ) + hyper.add_search( + experiment1, + search_config, + search_id="2", + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data("1"), pd.DataFrame) +""" + + +def test_attributes_result_errors_0(): + with pytest.raises(ValueError): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15) + hyper.run() + + hyper.best_para(experiment1) + + +def test_attributes_result_errors_1(): + with pytest.raises(ValueError): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15) + hyper.run() + + hyper.best_score(experiment1) + + +def test_attributes_result_errors_2(): + with pytest.raises(ValueError): + hyper = HillClimbingOptimizer() + hyper.add_search(experiment, search_config, n_iter=15) + hyper.run() + + hyper.search_data(experiment1) diff --git a/tests/test_api/test_search_spaces.py b/tests/test_api/test_search_spaces.py new file mode 100644 index 00000000..a8d1225e --- /dev/null +++ b/tests/test_api/test_search_spaces.py @@ -0,0 +1,185 @@ +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer +from hyperactive.experiment import BaseExperiment +from hyperactive.search_config import SearchConfig + + +class Experiment(BaseExperiment): + def objective_function(self, opt): + score = -opt["x1"] * opt["x1"] + return score + + +experiment = Experiment() + + +def test_search_space_0(): + search_config = SearchConfig( + x1=list(np.arange(0, 3, 1)), + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["x1"] in search_config["x1"] + + +def test_search_space_1(): + search_config = SearchConfig( + x1=list(np.arange(0, 0.003, 0.001)), + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["x1"] in search_config["x1"] + + +def test_search_space_2(): + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + str1=["0", "1", "2"], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["str1"] in search_config["str1"] + + +def test_search_space_3(): + def func1(): + pass + + def func2(): + pass + + def func3(): + pass + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + func1=[func1, func2, func3], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["func1"] in search_config["func1"] + + +def test_search_space_4(): + class class1: + pass + + class class2: + pass + + class class3: + pass + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + class1=[class1, class2, class3], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["class1"] in search_config["class1"] + + +def test_search_space_5(): + class class1: + def __init__(self): + pass + + class class2: + def __init__(self): + pass + + class class3: + def __init__(self): + pass + + def class_f1(): + return class1 + + def class_f2(): + return class2 + + def class_f3(): + return class3 + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + class1=[class_f1, class_f2, class_f3], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["class1"] in search_config["class1"] + + +def test_search_space_6(): + + def list_f1(): + return [0, 1] + + def list_f2(): + return [1, 0] + + search_config = SearchConfig( + x1=list(np.arange(0, 100, 1)), + list1=[list_f1, list_f2], + ) + + hyper = HillClimbingOptimizer() + hyper.add_search( + experiment, + search_config, + n_iter=15, + ) + hyper.run() + + assert isinstance(hyper.search_data(experiment), pd.DataFrame) + assert hyper.best_para(experiment)["list1"] in search_config["list1"] diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 00000000..8819d824 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,22 @@ +import copy +import pytest +import numpy as np +import pandas as pd + +from hyperactive.optimizers import HillClimbingOptimizer + + +from .test_setup import SphereFunction, search_config + + +objective_function = SphereFunction() + + +def test_callback_0(): + hyper = HillClimbingOptimizer() + hyper.add_search( + objective_function, + search_config, + n_iter=100, + ) + hyper.run() diff --git a/tests/test_setup/__init__.py b/tests/test_setup/__init__.py new file mode 100644 index 00000000..497df055 --- /dev/null +++ b/tests/test_setup/__init__.py @@ -0,0 +1,2 @@ +from .experiment import SphereFunction +from .search_config import search_config \ No newline at end of file diff --git a/tests/test_setup/experiment.py b/tests/test_setup/experiment.py new file mode 100644 index 00000000..5907e4ec --- /dev/null +++ b/tests/test_setup/experiment.py @@ -0,0 +1,11 @@ +import numpy as np + +from hyperactive.experiment import BaseExperiment + + +class SphereFunction(BaseExperiment): + def setup(self, n_dim=2): + self.n_dim = n_dim + + def objective_function(self, params): + return np.sum(params["x0"] ** 2 + params["x1"] ** 2) diff --git a/tests/test_setup/search_config.py b/tests/test_setup/search_config.py new file mode 100644 index 00000000..03e7e725 --- /dev/null +++ b/tests/test_setup/search_config.py @@ -0,0 +1,8 @@ +import numpy as np +from hyperactive.search_config import SearchConfig + + +search_config = SearchConfig( + x0=list(np.arange(2, 15, 1)), + x1=list(np.arange(2, 25, 2)), +) diff --git a/tests/integrations/sklearn/__init__.py b/tests_old/__init__.py similarity index 100% rename from tests/integrations/sklearn/__init__.py rename to tests_old/__init__.py diff --git a/tests/test_empty_output/__init__.py b/tests_old/_local_test_optimization_strategies/__init__.py similarity index 100% rename from tests/test_empty_output/__init__.py rename to tests_old/_local_test_optimization_strategies/__init__.py diff --git a/tests/_local_test_optimization_strategies/_parametrize.py b/tests_old/_local_test_optimization_strategies/_parametrize.py similarity index 100% rename from tests/_local_test_optimization_strategies/_parametrize.py rename to tests_old/_local_test_optimization_strategies/_parametrize.py diff --git a/tests/_local_test_optimization_strategies/_test_memory_warm_start.py b/tests_old/_local_test_optimization_strategies/_test_memory_warm_start.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_memory_warm_start.py rename to tests_old/_local_test_optimization_strategies/_test_memory_warm_start.py diff --git a/tests/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py b/tests_old/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py rename to tests_old/_local_test_optimization_strategies/_test_memory_warm_start_smbo.py diff --git a/tests/_local_test_optimization_strategies/_test_strategy_combinations.py b/tests_old/_local_test_optimization_strategies/_test_strategy_combinations.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_strategy_combinations.py rename to tests_old/_local_test_optimization_strategies/_test_strategy_combinations.py diff --git a/tests/_local_test_optimization_strategies/_test_strategy_multiprocessing.py b/tests_old/_local_test_optimization_strategies/_test_strategy_multiprocessing.py similarity index 100% rename from tests/_local_test_optimization_strategies/_test_strategy_multiprocessing.py rename to tests_old/_local_test_optimization_strategies/_test_strategy_multiprocessing.py diff --git a/tests/test_issues/__init__.py b/tests_old/_local_test_timings/__init__.py similarity index 100% rename from tests/test_issues/__init__.py rename to tests_old/_local_test_timings/__init__.py diff --git a/tests/_local_test_timings/_search_space_list.py b/tests_old/_local_test_timings/_search_space_list.py similarity index 100% rename from tests/_local_test_timings/_search_space_list.py rename to tests_old/_local_test_timings/_search_space_list.py diff --git a/tests/_local_test_timings/_test_memory.py b/tests_old/_local_test_timings/_test_memory.py similarity index 100% rename from tests/_local_test_timings/_test_memory.py rename to tests_old/_local_test_timings/_test_memory.py diff --git a/tests/_local_test_timings/_test_memory_warm_start.py b/tests_old/_local_test_timings/_test_memory_warm_start.py similarity index 100% rename from tests/_local_test_timings/_test_memory_warm_start.py rename to tests_old/_local_test_timings/_test_memory_warm_start.py diff --git a/tests/_local_test_timings/_test_memory_warm_start_n_jobs.py b/tests_old/_local_test_timings/_test_memory_warm_start_n_jobs.py similarity index 100% rename from tests/_local_test_timings/_test_memory_warm_start_n_jobs.py rename to tests_old/_local_test_timings/_test_memory_warm_start_n_jobs.py diff --git a/tests/_local_test_timings/_test_shared_memory.py b/tests_old/_local_test_timings/_test_shared_memory.py similarity index 100% rename from tests/_local_test_timings/_test_shared_memory.py rename to tests_old/_local_test_timings/_test_shared_memory.py diff --git a/tests/_local_test_timings/_test_warm_start.py b/tests_old/_local_test_timings/_test_warm_start.py similarity index 100% rename from tests/_local_test_timings/_test_warm_start.py rename to tests_old/_local_test_timings/_test_warm_start.py diff --git a/tests/_local_test_timings/_test_warm_start_n_jobs.py b/tests_old/_local_test_timings/_test_warm_start_n_jobs.py similarity index 100% rename from tests/_local_test_timings/_test_warm_start_n_jobs.py rename to tests_old/_local_test_timings/_test_warm_start_n_jobs.py diff --git a/tests/_test_examples.py b/tests_old/_test_examples.py similarity index 100% rename from tests/_test_examples.py rename to tests_old/_test_examples.py diff --git a/tests/test_optimization_strategies/__init__.py b/tests_old/integrations/__init__.py similarity index 100% rename from tests/test_optimization_strategies/__init__.py rename to tests_old/integrations/__init__.py diff --git a/tests/test_optimizers/__init__.py b/tests_old/integrations/sklearn/__init__.py similarity index 100% rename from tests/test_optimizers/__init__.py rename to tests_old/integrations/sklearn/__init__.py diff --git a/tests/integrations/sklearn/test_parametrize_with_checks.py b/tests_old/integrations/sklearn/test_parametrize_with_checks.py similarity index 100% rename from tests/integrations/sklearn/test_parametrize_with_checks.py rename to tests_old/integrations/sklearn/test_parametrize_with_checks.py diff --git a/tests/integrations/sklearn/test_sklearn_api.py b/tests_old/integrations/sklearn/test_sklearn_api.py similarity index 100% rename from tests/integrations/sklearn/test_sklearn_api.py rename to tests_old/integrations/sklearn/test_sklearn_api.py diff --git a/tests/test_callbacks.py b/tests_old/test_callbacks.py similarity index 100% rename from tests/test_callbacks.py rename to tests_old/test_callbacks.py diff --git a/tests/test_catch.py b/tests_old/test_catch.py similarity index 100% rename from tests/test_catch.py rename to tests_old/test_catch.py diff --git a/tests/test_constr_opt.py b/tests_old/test_constr_opt.py similarity index 100% rename from tests/test_constr_opt.py rename to tests_old/test_constr_opt.py diff --git a/tests/test_distribution.py b/tests_old/test_distribution.py similarity index 100% rename from tests/test_distribution.py rename to tests_old/test_distribution.py diff --git a/tests/test_early_stop.py b/tests_old/test_early_stop.py similarity index 100% rename from tests/test_early_stop.py rename to tests_old/test_early_stop.py diff --git a/tests/test_warm_starts/__init__.py b/tests_old/test_empty_output/__init__.py similarity index 100% rename from tests/test_warm_starts/__init__.py rename to tests_old/test_empty_output/__init__.py diff --git a/tests/test_empty_output/non_verbose.py b/tests_old/test_empty_output/non_verbose.py similarity index 100% rename from tests/test_empty_output/non_verbose.py rename to tests_old/test_empty_output/non_verbose.py diff --git a/tests/test_empty_output/test_empty_output.py b/tests_old/test_empty_output/test_empty_output.py similarity index 100% rename from tests/test_empty_output/test_empty_output.py rename to tests_old/test_empty_output/test_empty_output.py diff --git a/tests/test_empty_output/verbose.py b/tests_old/test_empty_output/verbose.py similarity index 100% rename from tests/test_empty_output/verbose.py rename to tests_old/test_empty_output/verbose.py diff --git a/tests/test_hyper_gradient_trafo.py b/tests_old/test_hyper_gradient_trafo.py similarity index 100% rename from tests/test_hyper_gradient_trafo.py rename to tests_old/test_hyper_gradient_trafo.py diff --git a/tests/test_initializers.py b/tests_old/test_initializers.py similarity index 100% rename from tests/test_initializers.py rename to tests_old/test_initializers.py diff --git a/tests_old/test_issues/__init__.py b/tests_old/test_issues/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_issues/test_issue_25.py b/tests_old/test_issues/test_issue_25.py similarity index 100% rename from tests/test_issues/test_issue_25.py rename to tests_old/test_issues/test_issue_25.py diff --git a/tests/test_issues/test_issue_29.py b/tests_old/test_issues/test_issue_29.py similarity index 100% rename from tests/test_issues/test_issue_29.py rename to tests_old/test_issues/test_issue_29.py diff --git a/tests/test_issues/test_issue_34.py b/tests_old/test_issues/test_issue_34.py similarity index 100% rename from tests/test_issues/test_issue_34.py rename to tests_old/test_issues/test_issue_34.py diff --git a/tests/test_max_score.py b/tests_old/test_max_score.py similarity index 100% rename from tests/test_max_score.py rename to tests_old/test_max_score.py diff --git a/tests/test_max_time.py b/tests_old/test_max_time.py similarity index 100% rename from tests/test_max_time.py rename to tests_old/test_max_time.py diff --git a/tests/test_obj_func_arg.py b/tests_old/test_obj_func_arg.py similarity index 100% rename from tests/test_obj_func_arg.py rename to tests_old/test_obj_func_arg.py diff --git a/tests_old/test_optimization_strategies/__init__.py b/tests_old/test_optimization_strategies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_optimization_strategies/_parametrize.py b/tests_old/test_optimization_strategies/_parametrize.py similarity index 100% rename from tests/test_optimization_strategies/_parametrize.py rename to tests_old/test_optimization_strategies/_parametrize.py diff --git a/tests/test_optimization_strategies/test_constr_opt.py b/tests_old/test_optimization_strategies/test_constr_opt.py similarity index 100% rename from tests/test_optimization_strategies/test_constr_opt.py rename to tests_old/test_optimization_strategies/test_constr_opt.py diff --git a/tests/test_optimization_strategies/test_early_stopping.py b/tests_old/test_optimization_strategies/test_early_stopping.py similarity index 100% rename from tests/test_optimization_strategies/test_early_stopping.py rename to tests_old/test_optimization_strategies/test_early_stopping.py diff --git a/tests/test_optimization_strategies/test_search_space_pruning.py b/tests_old/test_optimization_strategies/test_search_space_pruning.py similarity index 100% rename from tests/test_optimization_strategies/test_search_space_pruning.py rename to tests_old/test_optimization_strategies/test_search_space_pruning.py diff --git a/tests_old/test_optimizers/__init__.py b/tests_old/test_optimizers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_optimizers/_parametrize.py b/tests_old/test_optimizers/_parametrize.py similarity index 100% rename from tests/test_optimizers/_parametrize.py rename to tests_old/test_optimizers/_parametrize.py diff --git a/tests/test_optimizers/test_best_results.py b/tests_old/test_optimizers/test_best_results.py similarity index 100% rename from tests/test_optimizers/test_best_results.py rename to tests_old/test_optimizers/test_best_results.py diff --git a/tests/test_optimizers/test_gfo_wrapper.py b/tests_old/test_optimizers/test_gfo_wrapper.py similarity index 100% rename from tests/test_optimizers/test_gfo_wrapper.py rename to tests_old/test_optimizers/test_gfo_wrapper.py diff --git a/tests/test_optimizers/test_memory.py b/tests_old/test_optimizers/test_memory.py similarity index 100% rename from tests/test_optimizers/test_memory.py rename to tests_old/test_optimizers/test_memory.py diff --git a/tests/test_optimizers/test_optimization_strategies.py b/tests_old/test_optimizers/test_optimization_strategies.py similarity index 100% rename from tests/test_optimizers/test_optimization_strategies.py rename to tests_old/test_optimizers/test_optimization_strategies.py diff --git a/tests/test_pass_through.py b/tests_old/test_pass_through.py similarity index 100% rename from tests/test_pass_through.py rename to tests_old/test_pass_through.py diff --git a/tests/test_random_state.py b/tests_old/test_random_state.py similarity index 100% rename from tests/test_random_state.py rename to tests_old/test_random_state.py diff --git a/tests/test_results.py b/tests_old/test_results.py similarity index 100% rename from tests/test_results.py rename to tests_old/test_results.py diff --git a/tests/test_results_methods.py b/tests_old/test_results_methods.py similarity index 100% rename from tests/test_results_methods.py rename to tests_old/test_results_methods.py diff --git a/tests/test_search_spaces.py b/tests_old/test_search_spaces.py similarity index 100% rename from tests/test_search_spaces.py rename to tests_old/test_search_spaces.py diff --git a/tests_old/test_warm_starts/__init__.py b/tests_old/test_warm_starts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_warm_starts/test_memory_warm_start.py b/tests_old/test_warm_starts/test_memory_warm_start.py similarity index 100% rename from tests/test_warm_starts/test_memory_warm_start.py rename to tests_old/test_warm_starts/test_memory_warm_start.py diff --git a/tests/test_warm_starts/test_warm_start.py b/tests_old/test_warm_starts/test_warm_start.py similarity index 100% rename from tests/test_warm_starts/test_warm_start.py rename to tests_old/test_warm_starts/test_warm_start.py diff --git a/tests/test_warm_starts/test_warm_start_smbo.py b/tests_old/test_warm_starts/test_warm_start_smbo.py similarity index 100% rename from tests/test_warm_starts/test_warm_start_smbo.py rename to tests_old/test_warm_starts/test_warm_start_smbo.py