V5 api design

examples/v5_API_example/example.py

@@ -0,0 +1,39 @@
+import numpy as np
+from sklearn.datasets import load_diabetes
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.model_selection import cross_val_score
+
+from hyperactive.search_config import SearchConfig
+from hyperactive.optimizers import HillClimbingOptimizer
+
+from hyperactive.base import BaseExperiment
+
+
+class SklearnExperiment(BaseExperiment):
+    def setup(self, estimator, X, y, cv=5):
+        self.estimator = estimator
+        self.X = X
+        self.y = y
+        self.cv = cv
+
+    def _score(self, params):
+        model = self.estimator(**params)
+        scores = cross_val_score(model, self.X, self.y, cv=self.cv)
+        return scores.mean()
+
+
+data = load_diabetes()
+X, y = data.data, data.target
+
+
+search_config = SearchConfig(
+    max_depth=list(np.arange(2, 15, 1)),
+    min_samples_split=list(np.arange(2, 25, 2)),
+)
+
+experiment = SklearnExperiment()
+experiment.setup(DecisionTreeRegressor, X, y, cv=4)
+
+optimizer1 = HillClimbingOptimizer()
+optimizer1.add_search(experiment, search_config, n_iter=100, n_jobs=2)
+optimizer1.run()

examples/v5_API_example/experiments/keras.py

@@ -0,0 +1,68 @@
+from tensorflow import keras
+from sklearn.datasets import make_classification
+from sklearn.model_selection import train_test_split
+
+from hyperactive.base import BaseExperiment
+
+
+X, y = make_classification(n_samples=1000, n_features=20, random_state=42)
+X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)
+
+
+class KerasMultiLayerPerceptron(BaseExperiment):
+    """
+    A class for creating and evaluating a Keras-based Multi-Layer Perceptron (MLP) model.
+
+    This class inherits from BaseExperiment and is designed to build a simple MLP
+    using Keras, compile it with the Adam optimizer, and train it on the provided
+    training data. The model consists of one hidden dense layer with configurable
+    size and activation function, followed by an output layer with a sigmoid
+    activation for binary classification.
+
+    Attributes:
+        X_train (array-like): Training feature data.
+        X_val (array-like): Validation feature data.
+        y_train (array-like): Training target data.
+        y_val (array-like): Validation target data.
+
+    Methods:
+        _score(**params): Builds, compiles, and trains the MLP model using the
+        specified parameters for the hidden layer, and returns the validation
+        accuracy.
+    """
+
+    def __init__(self, X_train, X_val, y_train, y_val):
+        super().__init__()
+
+        self.X_train = X_train
+        self.X_val = X_val
+        self.y_train = y_train
+        self.y_val = y_val
+
+    def _score(self, **params):
+        dense_layer_0 = params["dense_layer_0"]
+        activation_layer_0 = params["activation_layer_0"]
+
+        model = keras.Sequential(
+            [
+                keras.layers.Dense(
+                    dense_layer_0,
+                    activation=activation_layer_0,
+                    input_shape=(20,),
+                ),
+                keras.layers.Dense(1, activation="sigmoid"),
+            ]
+        )
+        model.compile(
+            optimizer=keras.optimizers.Adam(learning_rate=0.01),
+            loss="binary_crossentropy",
+            metrics=["accuracy"],
+        )
+        model.fit(
+            self.X_train,
+            self.y_train,
+            batch_size=32,
+            epochs=10,
+            validation_data=(self.X_val, self.y_val),
+        )
+        return model.evaluate(X_val, y_val)[1]

examples/v5_API_example/experiments/sklearn.py

@@ -0,0 +1,67 @@
+from sklearn.model_selection import cross_val_score
+from sklearn.ensemble import GradientBoostingRegressor
+
+
+from hyperactive.base import BaseExperiment
+
+
+class SklearnExperiment(BaseExperiment):
+    """
+    Initializes the SklearnExperiment with the given estimator, data, and cross-validation settings.
+
+    Parameters
+    ----------
+    estimator : object
+        The machine learning estimator to be used for the experiment.
+    X : array-like
+        The input data for training the model.
+    y : array-like
+        The target values corresponding to the input data.
+    cv : int, optional
+        The number of cross-validation folds (default is 4).
+    """
+
+    def __init__(self, estimator, X, y, cv=4):
+        super().__init__()
+
+        self.estimator = estimator
+        self.X = X
+        self.y = y
+        self.cv = cv
+
+    def _score(self, **params):
+        model = self.estimator(**params)
+        scores = cross_val_score(model, self.X, self.y, cv=self.cv)
+        return scores.mean()
+
+
+class GradientBoostingExperiment(BaseExperiment):
+    """
+    A class for conducting experiments with Gradient Boosting Regressor using cross-validation.
+
+    This class inherits from BaseExperiment and allows users to perform experiments
+    with the GradientBoostingRegressor from sklearn. Users can specify the input
+    features, target values, and the number of cross-validation folds.
+
+    Attributes:
+        estimator (type): The regression model to be used, default is GradientBoostingRegressor.
+        X (array-like): The input features for the model.
+        y (array-like): The target values for the model.
+        cv (int): The number of cross-validation folds.
+
+    Methods:
+        _score(**params): Evaluates the model using cross-validation and returns the mean score.
+    """
+
+    def __init__(self, X, y, cv=4):
+        super().__init__()
+
+        self.estimator = GradientBoostingRegressor  # The user could also predefine the estimator
+        self.X = X
+        self.y = y
+        self.cv = cv
+
+    def _score(self, **params):
+        model = self.estimator(**params)
+        scores = cross_val_score(model, self.X, self.y, cv=self.cv)
+        return scores.mean()

examples/v5_API_example/experiments/test_function.py

@@ -0,0 +1,60 @@
+import numpy as np
+
+from hyperactive.base import BaseExperiment
+
+
+class SphereFunction(BaseExperiment):
+    """
+    A class representing a Sphere function experiment.
+
+    This class inherits from BaseExperiment and implements a simple
+    Sphere function, which is a common test function for optimization
+    algorithms. The function is defined as the sum of the squares of
+    its input parameters plus a constant.
+
+    Attributes:
+        const (float): A constant added to the function's result.
+        n_dim (int): The number of dimensions for the input parameters.
+
+    Methods:
+        _score(**params): Computes the Sphere function value for the
+        given parameters.
+    """
+
+    def __init__(self, const, n_dim=2):
+        super().__init__()
+
+        self.const = const
+        self.n_dim = n_dim
+
+    def _score(self, **params):
+        return np.sum(np.array(params) ** 2) + self.const
+
+
+class AckleyFunction(BaseExperiment):
+    """
+    A class representing the Ackley function, used as a benchmark for optimization algorithms.
+
+    Attributes:
+        A (float): A constant used in the calculation of the Ackley function.
+
+    Methods:
+        _score(**params): Computes the Ackley function value for given parameters 'x0' and 'x1'.
+
+    The Ackley function is a non-convex function used to test optimization algorithms.
+    """
+
+    def __init__(self, A):
+        super().__init__()
+        self.A = A
+
+    def _score(self, **params):
+        x = params["x0"]
+        y = params["x1"]
+
+        loss1 = -self.A * np.exp(-0.2 * np.sqrt(0.5 * (x * x + y * y)))
+        loss2 = -np.exp(0.5 * (np.cos(2 * np.pi * x) + np.cos(2 * np.pi * y)))
+        loss3 = np.exp(1)
+        loss4 = self.A
+
+        return -(loss1 + loss2 + loss3 + loss4)

examples/v5_API_example/optimizer.py

@@ -0,0 +1,42 @@
+import numpy as np
+from sklearn.datasets import load_diabetes
+from sklearn.tree import DecisionTreeRegressor
+
+
+from hyperactive.base.search_space_optional import SearchSpace
+from hyperactive.optimizers import (
+    HillClimbingOptimizer,
+    RandomRestartHillClimbingOptimizer,
+)
+
+from .experiments.test_function import SklearnExperiment
+
+
+data = load_diabetes()
+X, y = data.data, data.target
+
+
+search_space = {
+    "max_depth": list(np.arange(2, 15, 1)),
+    "min_samples_split": list(np.arange(2, 25, 2)),
+}
+
+""" optional way of defining search-space
+search_space = SearchSpace(
+    max_depth=list(np.arange(2, 15, 1)),
+    min_samples_split=list(np.arange(2, 25, 2)),
+)
+"""
+
+experiment = SklearnExperiment(DecisionTreeRegressor, X, y, cv=4)
+
+optimizer1 = HillClimbingOptimizer(n_iter=50)
+optimizer2 = RandomRestartHillClimbingOptimizer(n_iter=50, n_jobs=2)
+
+optimizer1.add_search(experiment, search_space)
+optimizer2.add_search(experiment, search_space)
+
+# not sure about this way of combining optimizers. Might not be intuitive what the plus means.
+hyper = optimizer1 + optimizer2
+
+hyper.run(max_time=5)

pyproject.toml

@@ -40,7 +40,8 @@ dependencies = [
   "numpy >=1.18.1, <3.0.0",
   "tqdm >=4.48.0, <5.0.0",
   "pandas <3.0.0",
-  "gradient-free-optimizers >=1.2.4, <2.0.0",
+  "gradient-free-optimizers >=1.5.0, <2.0.0",
+  "scikit-base <1.0.0",
 ]
 
 [project.optional-dependencies]

src/hyperactive/__init__.py

@@ -6,11 +6,3 @@
 
 __version__ = importlib.metadata.version("hyperactive")
 __license__ = "MIT"
-
-
-from .hyperactive import Hyperactive
-
-
-__all__ = [
-    "Hyperactive",
-]

src/hyperactive/experiment/__init__.py

@@ -0,0 +1,6 @@
+"""Base classes for optimizers and experiments."""
+
+from ._experiment import BaseExperiment
+from ._utility import add_callback, add_catch
+
+__all__ = ["BaseExperiment", "add_callback", "add_catch"]