Merge branch 'dev'

Author: SimonBlanke

.github/workflows/tests.yml

@@ -36,6 +36,7 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest coveralls codecov pytest-cov
         python -m pip install .
+        python -m pip install .[distribution]
                 
     - name: Test with pytest
       run: |

README.md

@@ -120,11 +120,13 @@ Hyperactive features a collection of optimization algorithms that can be used fo
           <ul>
             <li><a href="./examples/optimization_techniques/parallel_tempering.py">Parallel Tempering</a></li>
             <li><a href="./examples/optimization_techniques/particle_swarm_optimization.py">Particle Swarm Optimizer</li>
+            <li>Spiral Optimization</li>
             <li><a href="./examples/optimization_techniques/evolution_strategy.py">Evolution Strategy</a></li>
           </ul><br>
         <a><b>Sequential Methods:</b></a>
           <ul>
             <li><a href="./examples/optimization_techniques/bayesian_optimization.py">Bayesian Optimization</a></li>
+            <li>Lipschitz Optimization</li>
             <li><a href="./examples/optimization_techniques/tpe.py">Tree of Parzen Estimators</a></li>
             <li><a href="./examples/optimization_techniques/forest_optimization.py">Forest Optimizer</a>
             [<a href="#/./overview#references">dto</a>] </li>
@@ -378,8 +380,10 @@ hyper.run()
     - PatternSearch
     - ParallelTemperingOptimizer
     - ParticleSwarmOptimizer
+    - SpiralOptimization
     - EvolutionStrategyOptimizer
     - BayesianOptimizer
+    - LipschitzOptimizer
     - TreeStructuredParzenEstimators
     - ForestOptimizer
 

hyperactive/__init__.py

@@ -2,7 +2,7 @@
 # Email: simon.blanke@yahoo.com
 # License: MIT License
 
-__version__ = "4.2.0"
+__version__ = "4.3.0"
 __license__ = "MIT"
 
 

hyperactive/distribution.py

@@ -2,10 +2,6 @@
 # Email: simon.blanke@yahoo.com
 # License: MIT License
 
-import multiprocessing as mp
-import pathos.multiprocessing as pmp
-from joblib import Parallel, delayed
-
 
 def single_process(process_func, process_infos):
     results = [process_func(*info) for info in process_infos]
@@ -14,21 +10,38 @@ def single_process(process_func, process_infos):
 
 
 def multiprocessing_wrapper(process_func, process_infos, n_processes):
+    import multiprocessing as mp
+
     pool = mp.Pool(n_processes)
     results = pool.map(process_func, process_infos)
 
     return results
 
 
 def pathos_wrapper(process_func, search_processes_paras, n_processes, **kwargs):
+    import pathos.multiprocessing as pmp
+
     pool = pmp.Pool(n_processes, **kwargs)
     results = pool.map(process_func, search_processes_paras)
 
     return results
 
 
 def joblib_wrapper(process_func, search_processes_paras, n_processes, **kwargs):
+    from joblib import Parallel, delayed
+
     jobs = [delayed(process_func)(*info_dict) for info_dict in search_processes_paras]
     results = Parallel(n_jobs=n_processes, **kwargs)(jobs)
 
     return results
+
+
+def ray_wrapper(process_func, process_infos, n_processes, **kwargs):
+    import ray
+    from ray.util.multiprocessing import Pool
+
+    # ray.init(log_to_driver=False)
+    pool = Pool(n_processes)
+    results = pool.map(process_func, process_infos)
+
+    return results

hyperactive/hyperactive.py

@@ -4,7 +4,6 @@
 
 
 import multiprocessing as mp
-from tqdm import tqdm
 
 from .optimizers import RandomSearchOptimizer
 from .run_search import run_search

hyperactive/optimizers/__init__.py

@@ -17,8 +17,11 @@
     PatternSearch,
     ParallelTemperingOptimizer,
     ParticleSwarmOptimizer,
+    SpiralOptimization,
     EvolutionStrategyOptimizer,
     BayesianOptimizer,
+    LipschitzOptimizer,
+    DirectAlgorithm,
     TreeStructuredParzenEstimators,
     ForestOptimizer,
     EnsembleOptimizer,
@@ -39,8 +42,11 @@
     "PatternSearch",
     "ParallelTemperingOptimizer",
     "ParticleSwarmOptimizer",
+    "SpiralOptimization",
     "EvolutionStrategyOptimizer",
     "BayesianOptimizer",
+    "LipschitzOptimizer",
+    "DirectAlgorithm",
     "TreeStructuredParzenEstimators",
     "ForestOptimizer",
     "EnsembleOptimizer",

hyperactive/optimizers/base_optimizer.py

@@ -0,0 +1,30 @@
+# Author: Simon Blanke
+# Email: simon.blanke@yahoo.com
+# License: MIT License
+
+
+class BaseOptimizer:
+    def __init__(self):
+        pass
+
+    def _add_result_attributes(
+        self,
+        best_para,
+        best_score,
+        best_since_iter,
+        eval_times,
+        iter_times,
+        positions,
+        search_data,
+        memory_values_df,
+        random_seed,
+    ):
+        self.best_para = best_para
+        self.best_score = best_score
+        self.best_since_iter = best_since_iter
+        self.eval_times = eval_times
+        self.iter_times = iter_times
+        self.positions = positions
+        self.search_data = search_data
+        self.memory_values_df = memory_values_df
+        self.random_seed = random_seed

hyperactive/optimizers/exp_opt_wrapper.py

@@ -96,16 +96,29 @@ def get_list_positions(self, list1_values, search_dim):
         list_positions = []
 
         for value2 in list1_values:
-            try:
-                list_positions.append(search_dim.index(value2))
-            except:
-                raise ValueError
+            list_positions.append(search_dim.index(value2))
 
         return list_positions
 
     def values2positions(self, values, search_dim):
         return np.array(search_dim).searchsorted(values)
 
+    def positions2results(self, positions):
+        results_dict = {}
+
+        for para_name in self.s_space.dim_keys:
+            values_list = self.s_space[para_name]
+            pos_ = positions[para_name].values
+            values_ = [values_list[idx] for idx in pos_]
+            results_dict[para_name] = values_
+
+        results = pd.DataFrame.from_dict(results_dict)
+
+        diff_list = np.setdiff1d(positions.columns, results.columns)
+        results[diff_list] = positions[diff_list]
+
+        return results
+
     def conv_memory_warm_start(self, results):
         if results is None:
             return results

hyperactive/optimizers/hyper_gradient_conv.py

@@ -2,68 +2,16 @@
 # Email: simon.blanke@yahoo.com
 # License: MIT License
 
-import copy
-import inspect
 import numpy as np
 import pandas as pd
 
 
 from .objective_function import ObjectiveFunction
 from .hyper_gradient_conv import HyperGradientConv
+from .base_optimizer import BaseOptimizer
 
 
-class TrafoClass:
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def _convert_args2gfo(self, memory_warm_start):
-        memory_warm_start = self.hg_conv.conv_memory_warm_start(memory_warm_start)
-
-        return memory_warm_start
-
-    def _positions2results(self, positions):
-        results_dict = {}
-
-        for para_name in self.conv.para_names:
-            values_list = self.s_space[para_name]
-            pos_ = positions[para_name].values
-            values_ = [values_list[idx] for idx in pos_]
-            results_dict[para_name] = values_
-
-        results = pd.DataFrame.from_dict(results_dict)
-
-        diff_list = np.setdiff1d(positions.columns, results.columns)
-        results[diff_list] = positions[diff_list]
-
-        return results
-
-    def _convert_results2hyper(self):
-        self.eval_times = np.array(self._optimizer.eval_times).sum()
-        self.iter_times = np.array(self._optimizer.iter_times).sum()
-
-        if self._optimizer.best_para is not None:
-            value = self.hg_conv.para2value(self._optimizer.best_para)
-            position = self.hg_conv.position2value(value)
-            best_para = self.hg_conv.value2para(position)
-
-            self.best_para = best_para
-        else:
-            self.best_para = None
-
-        self.best_score = self._optimizer.best_score
-        self.positions = self._optimizer.search_data
-
-        self.search_data = self._positions2results(self.positions)
-
-        results_dd = self._optimizer.search_data.drop_duplicates(
-            subset=self.s_space.dim_keys, keep="first"
-        )
-        self.memory_values_df = results_dd[
-            self.s_space.dim_keys + ["score"]
-        ].reset_index(drop=True)
-
-
-class _BaseOptimizer_(TrafoClass):
+class HyperOptimizer(BaseOptimizer):
     def __init__(self, **opt_params):
         super().__init__()
         self.opt_params = opt_params
@@ -104,6 +52,31 @@ def setup_search(
         else:
             self.verbosity = []
 
+    def convert_results2hyper(self):
+        self.eval_times = np.array(self.opt_algo.eval_times).sum()
+        self.iter_times = np.array(self.opt_algo.iter_times).sum()
+
+        if self.opt_algo.best_para is not None:
+            value = self.hg_conv.para2value(self.opt_algo.best_para)
+            position = self.hg_conv.position2value(value)
+            best_para = self.hg_conv.value2para(position)
+
+            self.best_para = best_para
+        else:
+            self.best_para = None
+
+        self.best_score = self.opt_algo.best_score
+        self.positions = self.opt_algo.search_data
+
+        self.search_data = self.hg_conv.positions2results(self.positions)
+
+        results_dd = self.opt_algo.search_data.drop_duplicates(
+            subset=self.s_space.dim_keys, keep="first"
+        )
+        self.memory_values_df = results_dd[
+            self.s_space.dim_keys + ["score"]
+        ].reset_index(drop=True)
+
     def _setup_process(self, nth_process):
         self.nth_process = nth_process
 
@@ -118,33 +91,33 @@ def _setup_process(self, nth_process):
                 self.opt_params["warm_start_smbo"]
             )
 
-        self._optimizer = self._OptimizerClass(
+        self.opt_algo = self._OptimizerClass(
             search_space=search_space_positions,
             initialize=initialize,
             random_state=self.random_state,
             nth_process=nth_process,
             **self.opt_params
         )
 
-        self.conv = self._optimizer.conv
+        self.conv = self.opt_algo.conv
 
     def search(self, nth_process):
         self._setup_process(nth_process)
 
         gfo_wrapper_model = ObjectiveFunction(
             objective_function=self.objective_function,
-            optimizer=self._optimizer,
+            optimizer=self.opt_algo,
             callbacks=self.callbacks,
             catch=self.catch,
             nth_process=self.nth_process,
         )
         gfo_wrapper_model.pass_through = self.pass_through
 
-        memory_warm_start = self._convert_args2gfo(self.memory_warm_start)
+        memory_warm_start = self.hg_conv.conv_memory_warm_start(self.memory_warm_start)
 
         gfo_objective_function = gfo_wrapper_model(self.s_space())
 
-        self._optimizer.search(
+        self.opt_algo.search(
             objective_function=gfo_objective_function,
             n_iter=self.n_iter,
             max_time=self.max_time,
@@ -155,5 +128,16 @@ def search(self, nth_process):
             verbosity=self.verbosity,
         )
 
-        self._convert_results2hyper()
-        self.p_bar = self._optimizer.p_bar
+        self.convert_results2hyper()
+
+        self._add_result_attributes(
+            self.best_para,
+            self.best_score,
+            self.opt_algo.p_bar._best_since_iter,
+            self.eval_times,
+            self.iter_times,
+            self.positions,
+            self.search_data,
+            self.memory_values_df,
+            self.opt_algo.random_seed,
+        )