[ENH] parallelization backends for grid and random search

Makefile

@@ -90,7 +90,7 @@ install-no-extras-for-test:
 	python -m pip install .[test]
 
 install-all-extras-for-test:
-	python -m pip install .[all_extras,sktime-integration,test]
+	python -m pip install .[all_extras,test,test_parallel_backends,sktime-integration]
 
 install-editable:
 	pip install -e .

examples/optimization_techniques/grid_search_sk.py

@@ -0,0 +1,33 @@
+"""
+GridSearchSk Optimizer Example
+
+This example demonstrates how to use the GridSearchSk optimizer with
+loky backend for parallel execution.
+"""
+
+import numpy as np
+from hyperactive.opt import GridSearchSk
+from hyperactive.experiment.toy import Sphere
+
+# Define the optimization problem using a toy sphere function
+# The sphere function f(x,y) = x² + y² has its minimum at (0,0)
+sphere_experiment = Sphere(n_dim=2)
+
+# Define parameter grid - creates a 9x9 = 81 point grid
+param_grid = {
+    "x0": np.linspace(-2, 2, 9),
+    "x1": np.linspace(-2, 2, 9),
+}
+
+# Grid search with parallel execution using loky backend
+grid_search = GridSearchSk(
+    param_grid=param_grid,
+    backend="loky",  # Use loky backend for parallelization
+    backend_params={
+        "n_jobs": -1,  # Use all available CPU cores
+    },
+    experiment=sphere_experiment,
+)
+
+best_params = grid_search.run()
+print(f"Best params: {best_params}, Score: {grid_search.best_score_:.6f}")

examples/optimization_techniques/random_search_sk.py

@@ -0,0 +1,44 @@
+"""
+RandomSearchSk Optimizer Example
+
+This example demonstrates how to use the RandomSearchSk optimizer with
+sklearn integration and threading backend for parallel execution.
+"""
+
+import numpy as np
+from scipy.stats import uniform, randint
+from sklearn.datasets import load_iris
+from sklearn.svm import SVC
+from hyperactive.opt import RandomSearchSk
+from hyperactive.experiment.integrations import SklearnCvExperiment
+
+# Load iris dataset and create sklearn experiment
+X, y = load_iris(return_X_y=True)
+sklearn_experiment = SklearnCvExperiment(
+    estimator=SVC(),
+    X=X,
+    y=y,
+)
+
+# Define parameter distributions for random search
+# Mix of discrete and continuous distributions
+param_distributions = {
+    "C": uniform(loc=0.1, scale=10),  # Continuous uniform distribution
+    "gamma": ["scale", "auto"] + list(np.logspace(-4, 1, 20)),  # Mixed discrete/continuous
+    "kernel": ["rbf", "poly", "sigmoid"],  # Discrete choices
+}
+
+# Random search with threading backend
+random_search = RandomSearchSk(
+    param_distributions=param_distributions,
+    n_iter=30,  # Number of random samples
+    random_state=42,  # For reproducible results
+    backend="threading",  # Use threading backend for parallelization
+    backend_params={
+        "n_jobs": 2,  # Use 2 threads
+    },
+    experiment=sklearn_experiment,
+)
+
+best_params = random_search.run()
+print(f"Best params: {best_params}, Score: {random_search.best_score_:.4f}")

pyproject.toml

@@ -65,6 +65,11 @@ test = [
   "torch",
   "tf_keras",
 ]
+test_parallel_backends = [
+  "dask",
+  "joblib",
+  'ray >=2.40.0; python_version < "3.13"',
+]
 all_extras = [
   "hyperactive[integrations]",
   "optuna<5",

src/hyperactive/opt/_common.py

@@ -0,0 +1,16 @@
+"""Common functions used by multiple optimizers."""
+
+__all__ = ["_score_params"]
+
+
+def _score_params(params, meta):
+    """Score parameters, used in parallelization."""
+    meta = meta.copy()
+    experiment = meta["experiment"]
+    error_score = meta["error_score"]
+
+    try:
+        return experiment(**params)
+    except Exception:  # noqa: B904
+        # Catch all exceptions and assign error_score
+        return error_score

src/hyperactive/opt/gridsearch/_sk.py

@@ -1,4 +1,5 @@
 """Grid search optimizer."""
+
 # copyright: hyperactive developers, MIT License (see LICENSE file)
 
 from collections.abc import Sequence
@@ -7,6 +8,8 @@
 from sklearn.model_selection import ParameterGrid
 
 from hyperactive.base import BaseOptimizer
+from hyperactive.opt._common import _score_params
+from hyperactive.utils.parallel import parallelize
 
 
 class GridSearchSk(BaseOptimizer):
@@ -17,8 +20,45 @@ class GridSearchSk(BaseOptimizer):
     param_grid : dict[str, list]
         The search space to explore. A dictionary with parameter
         names as keys and a numpy array as values.
+
     error_score : float, default=np.nan
         The score to assign if an error occurs during the evaluation of a parameter set.
+
+    backend : {"dask", "loky", "multiprocessing", "threading", "ray"}, default = "None".
+        Parallelization backend to use in the search process.
+
+        - "None": executes loop sequentally, simple list comprehension
+        - "loky", "multiprocessing" and "threading": uses ``joblib.Parallel`` loops
+        - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``
+        - "dask": uses ``dask``, requires ``dask`` package in environment
+        - "ray": uses ``ray``, requires ``ray`` package in environment
+
+    backend_params : dict, optional
+        additional parameters passed to the backend as config.
+        Directly passed to ``utils.parallel.parallelize``.
+        Valid keys depend on the value of ``backend``:
+
+        - "None": no additional parameters, ``backend_params`` is ignored
+        - "loky", "multiprocessing" and "threading": default ``joblib`` backends
+          any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+          with the exception of ``backend`` which is directly controlled by ``backend``.
+          If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+          will default to ``joblib`` defaults.
+        - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``.
+          any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+          ``backend`` must be passed as a key of ``backend_params`` in this case.
+          If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+          will default to ``joblib`` defaults.
+        - "dask": any valid keys for ``dask.compute`` can be passed, e.g., ``scheduler``
+
+        - "ray": The following keys can be passed:
+
+            - "ray_remote_args": dictionary of valid keys for ``ray.init``
+            - "shutdown_ray": bool, default=True; False prevents ``ray`` from shutting
+                down after parallelization.
+            - "logger_name": str, default="ray"; name of the logger to use.
+            - "mute_warnings": bool, default=False; if True, suppresses warnings
+
     experiment : BaseExperiment, optional
         The experiment to optimize parameters for.
         Optional, can be passed later via ``set_params``.
@@ -53,17 +93,29 @@ class GridSearchSk(BaseOptimizer):
 
     Best parameters can also be accessed via the attributes:
     >>> best_params = grid_search.best_params_
+
+    To parallelize the search, set the ``backend`` and ``backend_params``:
+    >>> grid_search = GridSearch(
+    ...     param_grid,
+    ...     backend="joblib",
+    ...     backend_params={"n_jobs": -1},
+    ...     experiment=sklearn_exp,
+    ... )
     """
 
     def __init__(
         self,
         param_grid=None,
         error_score=np.nan,
+        backend="None",
+        backend_params=None,
         experiment=None,
     ):
         self.experiment = experiment
         self.param_grid = param_grid
         self.error_score = error_score
+        self.backend = backend
+        self.backend_params = backend_params
 
         super().__init__()
 
@@ -91,19 +143,30 @@ def _check_param_grid(self, param_grid):
                         "to be a non-empty sequence."
                     )
 
-    def _solve(self, experiment, param_grid, error_score):
+    def _solve(
+        self,
+        experiment,
+        param_grid,
+        error_score,
+        backend,
+        backend_params,
+    ):
         """Run the optimization search process."""
         self._check_param_grid(param_grid)
         candidate_params = list(ParameterGrid(param_grid))
 
-        scores = []
-        for candidate_param in candidate_params:
-            try:
-                score = experiment(**candidate_param)
-            except Exception:  # noqa: B904
-                # Catch all exceptions and assign error_score
-                score = error_score
-            scores.append(score)
+        meta = {
+            "experiment": experiment,
+            "error_score": error_score,
+        }
+
+        scores = parallelize(
+            fun=_score_params,
+            iter=candidate_params,
+            meta=meta,
+            backend=backend,
+            backend_params=backend_params,
+        )
 
         best_index = np.argmin(scores)
         best_params = candidate_params[best_index]
@@ -170,4 +233,15 @@ def get_test_params(cls, parameter_set="default"):
             "param_grid": param_grid,
         }
 
-        return [params_sklearn, params_ackley]
+        params = [params_sklearn, params_ackley]
+
+        from hyperactive.utils.parallel import _get_parallel_test_fixtures
+
+        parallel_fixtures = _get_parallel_test_fixtures()
+
+        for x in parallel_fixtures:
+            new_ackley = params_ackley.copy()
+            new_ackley.update(x)
+            params.append(new_ackley)
+
+        return params