Add optimization algorithms from GFO (hyperactive-project#127)

This PR adds all optimization algorithms from GFO in the new
`hyperactive v5` API, to the `opt` module.

---------

Co-authored-by: Franz Király <fkiraly@gcos.ai>

Author: SimonBlanke

Makefile

@@ -36,7 +36,7 @@ test-search_space:
 	done
 
 test-pytest:
-	python -m pytest --durations=10 -x -p  no:warnings tests/; \
+	python -m pytest --durations=10 -x -p  no:warnings tests/ src/hyperactive/; \
 
 test-timings:
 	cd tests/_local_test_timings; \

scripts/__init__.py

@@ -0,0 +1,63 @@
+import os
+from pathlib import Path
+
+# List of algorithm names and corresponding class names
+algo_info = [
+    ("downhill_simplex", "DownhillSimplexOptimizer"),
+    ("simulated_annealing", "SimulatedAnnealingOptimizer"),
+    ("direct_algorithm", "DirectAlgorithm"),
+    ("lipschitz_optimization", "LipschitzOptimizer"),
+    ("pattern_search", "PatternSearch"),
+    ("random_restart_hill_climbing", "RandomRestartHillClimbingOptimizer"),
+    ("random_search", "RandomSearchOptimizer"),
+    ("powells_method", "PowellsMethod"),
+    ("differential_evolution", "DifferentialEvolutionOptimizer"),
+    ("evolution_strategy", "EvolutionStrategyOptimizer"),
+    ("genetic_algorithm", "GeneticAlgorithmOptimizer"),
+    ("parallel_tempering", "ParallelTemperingOptimizer"),
+    ("particle_swarm_optimization", "ParticleSwarmOptimizer"),
+    ("spiral_optimization", "SpiralOptimization"),
+    ("bayesian_optimization", "BayesianOptimizer"),
+    ("forest_optimizer", "ForestOptimizer"),
+    ("tree_structured_parzen_estimators", "TreeStructuredParzenEstimators"),
+]
+
+BASE_DIR = Path("generated_opt_algos")
+
+
+# Template for the Python class file
+def create_class_file_content(class_name: str) -> str:
+    return f'''from hyperactive.opt._adapters._gfo import _BaseGFOadapter
+
+
+class {class_name}(_BaseGFOadapter):
+
+    def _get_gfo_class(self):
+        """Get the GFO class to use.
+
+        Returns
+        -------
+        class
+            The GFO class to use. One of the concrete GFO classes
+        """
+        from gradient_free_optimizers import {class_name}
+
+        return {class_name}
+'''
+
+
+# Main generation loop
+for name, class_name in algo_info:
+    algo_folder = BASE_DIR / name
+    algo_folder.mkdir(parents=True, exist_ok=True)
+
+    init_file = algo_folder / "__init__.py"
+    class_file = algo_folder / f"_{name}.py"
+
+    # Create __init__.py (empty)
+    init_file.touch(exist_ok=True)
+
+    # Write the optimizer class file
+    class_file.write_text(create_class_file_content(class_name))
+
+print(f"Generated {len(algo_info)} folders in {BASE_DIR.resolve()}")

scripts/_generator.py

@@ -1,4 +1,5 @@
 """Base class for optimizer."""
+
 # copyright: hyperactive developers, MIT License (see LICENSE file)
 
 from skbase.base import BaseObject

src/hyperactive/base/_optimizer.py

@@ -1,14 +1,54 @@
 """Individual optimization algorithms."""
+
 # copyright: hyperactive developers, MIT License (see LICENSE file)
 
 from hyperactive.opt.gridsearch import GridSearchSk
-from hyperactive.opt.hillclimbing import HillClimbing
-from hyperactive.opt.hillclimbing_repulsing import HillClimbingRepulsing
-from hyperactive.opt.hillclimbing_stochastic import HillClimbingStochastic
+from .gfo import (
+    HillClimbing,
+    StochasticHillClimbing,
+    RepulsingHillClimbing,
+    SimulatedAnnealing,
+    DownhillSimplexOptimizer,
+    RandomSearch,
+    GridSearch,
+    RandomRestartHillClimbing,
+    PowellsMethod,
+    PatternSearch,
+    LipschitzOptimizer,
+    DirectAlgorithm,
+    ParallelTempering,
+    ParticleSwarmOptimizer,
+    SpiralOptimization,
+    GeneticAlgorithm,
+    EvolutionStrategy,
+    DifferentialEvolution,
+    BayesianOptimizer,
+    TreeStructuredParzenEstimators,
+    ForestOptimizer,
+)
+
 
 __all__ = [
     "GridSearchSk",
     "HillClimbing",
-    "HillClimbingRepulsing",
-    "HillClimbingStochastic",
+    "RepulsingHillClimbing",
+    "StochasticHillClimbing",
+    "SimulatedAnnealing",
+    "DownhillSimplexOptimizer",
+    "RandomSearch",
+    "GridSearch",
+    "RandomRestartHillClimbing",
+    "PowellsMethod",
+    "PatternSearch",
+    "LipschitzOptimizer",
+    "DirectAlgorithm",
+    "ParallelTempering",
+    "ParticleSwarmOptimizer",
+    "SpiralOptimization",
+    "GeneticAlgorithm",
+    "EvolutionStrategy",
+    "DifferentialEvolution",
+    "BayesianOptimizer",
+    "TreeStructuredParzenEstimators",
+    "ForestOptimizer",
 ]

src/hyperactive/opt/__init__.py

@@ -1,4 +1,5 @@
 """Adapter for gfo package."""
+
 # copyright: hyperactive developers, MIT License (see LICENSE file)
 
 from hyperactive.base import BaseOptimizer
@@ -40,9 +41,7 @@ def _get_gfo_class(self):
         class
             The GFO class to use. One of the concrete GFO classes
         """
-        raise NotImplementedError(
-            "This method should be implemented in a subclass."
-        )
+        raise NotImplementedError("This method should be implemented in a subclass.")
 
     def get_search_config(self):
         """Get the search configuration.
@@ -55,8 +54,63 @@ def get_search_config(self):
         search_config = super().get_search_config()
         search_config["initialize"] = self._initialize
         del search_config["verbose"]
+
+        search_config = self._handle_gfo_defaults(search_config)
+
+        search_config["search_space"] = self._to_dict_np(search_config["search_space"])
+
+        return search_config
+
+    def _handle_gfo_defaults(self, search_config):
+        """Handle default values for GFO search configuration.
+
+        Temporary measure until GFO handles defaults gracefully.
+
+        Parameters
+        ----------
+        search_config : dict with str keys
+            The search configuration dictionary to handle defaults for.
+
+        Returns
+        -------
+        search_config : dict with str keys
+            The search configuration dictionary with defaults handled.
+        """
+        if "sampling" in search_config and search_config["sampling"] is None:
+            search_config["sampling"] = {"random": 1000000}
+
+        if "tree_para" in search_config and search_config["tree_para"] is None:
+            search_config["tree_para"] = {"n_estimators": 100}
+
         return search_config
 
+    def _to_dict_np(self, search_space):
+        """Coerce the search space to a format suitable for gfo optimizers.
+
+        gfo expects dicts of numpy arrays, not lists.
+        This method coerces lists or tuples in the search space to numpy arrays.
+
+        Parameters
+        ----------
+        search_space : dict with str keys and iterable values
+            The search space to coerce.
+
+        Returns
+        -------
+        dict with str keys and 1D numpy arrays as values
+            The coerced search space.
+        """
+        import numpy as np
+
+        def coerce_to_numpy(arr):
+            """Coerce a list or tuple to a numpy array."""
+            if not isinstance(arr, np.ndarray):
+                return np.array(arr)
+            return arr
+        
+        coerced_search_space = {k: coerce_to_numpy(v) for k, v in search_space.items()}
+        return coerced_search_space
+
     def _run(self, experiment, **search_config):
         """Run the optimization search process.
         Parameters
@@ -75,15 +129,15 @@ def _run(self, experiment, **search_config):
         max_time = search_config.pop("max_time", None)
 
         gfo_cls = self._get_gfo_class()
-        hcopt = gfo_cls(**search_config)
+        gfopt = gfo_cls(**search_config)
 
         with StdoutMute(active=not self.verbose):
-            hcopt.search(
+            gfopt.search(
                 objective_function=experiment.score,
                 n_iter=n_iter,
                 max_time=max_time,
             )
-        best_params = hcopt.best_para
+        best_params = gfopt.best_para
         return best_params
 
     @classmethod
@@ -143,5 +197,12 @@ def get_test_params(cls, parameter_set="default"):
             },
             "n_iter": 100,
         }
-        
-        return [params_sklearn, params_ackley]
+        params_ackley_list = {
+            "experiment": ackley_exp,
+            "search_space": {
+                "x0": list(np.linspace(-5, 5, 10)),
+                "x1": list(np.linspace(-5, 5, 10)),
+            },
+            "n_iter": 100,
+        }
+        return [params_sklearn, params_ackley, params_ackley_list]

src/hyperactive/opt/_adapters/_gfo.py

@@ -0,0 +1,50 @@
+"""Individual optimization algorithms."""
+
+# copyright: hyperactive developers, MIT License (see LICENSE file)
+
+from ._hillclimbing import HillClimbing
+from ._stochastic_hillclimbing import StochasticHillClimbing
+from ._repulsing_hillclimbing import RepulsingHillClimbing
+from ._simulated_annealing import SimulatedAnnealing
+from ._downhill_simplex import DownhillSimplexOptimizer
+from ._random_search import RandomSearch
+from ._grid_search import GridSearch
+from ._random_restart_hill_climbing import RandomRestartHillClimbing
+from ._powells_method import PowellsMethod
+from ._pattern_search import PatternSearch
+from ._lipschitz_optimization import LipschitzOptimizer
+from ._direct_algorithm import DirectAlgorithm
+from ._parallel_tempering import ParallelTempering
+from ._particle_swarm_optimization import ParticleSwarmOptimizer
+from ._spiral_optimization import SpiralOptimization
+from ._genetic_algorithm import GeneticAlgorithm
+from ._evolution_strategy import EvolutionStrategy
+from ._differential_evolution import DifferentialEvolution
+from ._bayesian_optimization import BayesianOptimizer
+from ._tree_structured_parzen_estimators import TreeStructuredParzenEstimators
+from ._forest_optimizer import ForestOptimizer
+
+
+__all__ = [
+    "HillClimbing",
+    "RepulsingHillClimbing",
+    "StochasticHillClimbing",
+    "SimulatedAnnealing",
+    "DownhillSimplexOptimizer",
+    "RandomSearch",
+    "GridSearch",
+    "RandomRestartHillClimbing",
+    "PowellsMethod",
+    "PatternSearch",
+    "LipschitzOptimizer",
+    "DirectAlgorithm",
+    "ParallelTempering",
+    "ParticleSwarmOptimizer",
+    "SpiralOptimization",
+    "GeneticAlgorithm",
+    "EvolutionStrategy",
+    "DifferentialEvolution",
+    "BayesianOptimizer",
+    "TreeStructuredParzenEstimators",
+    "ForestOptimizer",
+]