Make CostFunc return self.invalid_return_value instead of sys.float_info.max

Author: stijnh

kernel_tuner/runners/parallel.py

@@ -9,7 +9,7 @@
 from kernel_tuner.runners.runner import Runner
 from kernel_tuner.util import (
     Timer,
-    disable_benchmark_timings,
+    copy_without_benchmark_timings,
     ErrorConfig,
     TuningBudget,
     print_config_output,
@@ -313,7 +313,7 @@ def run(self, parameter_space, tuning_options) -> List[Optional[dict]]:
             if key in tuning_options.cache:
                 # We must disable the timings as otherwise these will counted
                 # as part of the total_compile/benchmark/verification_time
-                result = disable_benchmark_timings(tuning_options.cache[key])
+                result = copy_without_benchmark_timings(tuning_options.cache[key])
 
                 # recompute matrics for this entry
                 result = process_metrics(result, metrics)
@@ -369,7 +369,7 @@ def run(self, parameter_space, tuning_options) -> List[Optional[dict]]:
         # as otherwise we would count them multiple times in the total
         for i, j in duplicate_entries:
             if results[j]:
-                results[i] = disable_benchmark_timings(results[j])
+                results[i] = copy_without_benchmark_timings(results[j])
 
         # Count the number of valid results
         num_valid_results = sum(bool(r) for r in results)
@@ -388,7 +388,7 @@ def run(self, parameter_space, tuning_options) -> List[Optional[dict]]:
             for result in results:
                 if result:
                     # Time must be in ms
-                    result["strategy_time"] = strategy_time / num_valid_results
-                    result["framework_time"] = framework_time / num_valid_results
+                    result["strategy_time"] = 1000 * strategy_time / num_valid_results
+                    result["framework_time"] = 1000 * framework_time / num_valid_results
 
         return results

kernel_tuner/runners/runner.py

@@ -13,7 +13,8 @@ def __init__(self):
         self.timer = Timer()
         self.accumulated_strategy_time = 0
 
-    def add_strategy_time(self, seconds):
+    def add_strategy_time(self, seconds: float):
+        """ Notify this runner of the amount of time spent by the search strategy."""
         self.accumulated_strategy_time += seconds
 
     def shutdown(self):

kernel_tuner/runners/sequential.py

@@ -5,7 +5,7 @@
 
 from kernel_tuner.core import DeviceInterface
 from kernel_tuner.runners.runner import Runner
-from kernel_tuner.util import ErrorConfig, Timer, print_config_output, process_metrics, store_cache, disable_benchmark_timings
+from kernel_tuner.util import ErrorConfig, Timer, print_config_output, process_metrics, store_cache, copy_without_benchmark_timings
 
 
 class SequentialRunner(Runner):
@@ -63,6 +63,7 @@ def run(self, parameter_space, tuning_options):
 
         results = []
         worker_time = 0
+        warmup_time = 0
 
         # iterate over parameter space
         for element in parameter_space:
@@ -71,18 +72,16 @@ def run(self, parameter_space, tuning_options):
             if tuning_options.budget.is_done():
                 results.append(None)
                 continue
-            
+
             tuning_options.budget.add_evaluations(1)
             params = dict(zip(tuning_options.tune_params.keys(), element))
-
             result = None
-            warmup_time = 0
 
             # check if configuration is in the cache
             x_int = ",".join([str(i) for i in element])
             if tuning_options.cache and x_int in tuning_options.cache:
                 cache_entry = tuning_options.cache[x_int]
-                params.update(disable_benchmark_timings(cache_entry))
+                params.update(copy_without_benchmark_timings(cache_entry))
             else:
                 # attempt to warmup the GPU by running the first config in the parameter space and ignoring the result
                 if not self.warmed_up:
@@ -136,6 +135,7 @@ def run(self, parameter_space, tuning_options):
             # Amortize the time over all the results
             for result in results:
                 if result:
+                    # Time must be in ms
                     result["strategy_time"] = 1000 * strategy_time / num_valid_results
                     result["framework_time"] = 1000 * framework_time / num_valid_results
 

kernel_tuner/runners/simulation.py

@@ -60,7 +60,7 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob
 
     def get_device_info(self):
         return self.dev
-    
+
     def get_environment(self, tuning_options):
         env = self.dev.get_environment()
         env["simulation"] = True
@@ -90,7 +90,7 @@ def run(self, parameter_space, tuning_options):
             if tuning_options.budget.is_done():
                 results.append(None)
                 continue
-            
+
             # check if element is in the cache
             key = ",".join([str(i) for i in element])
 
@@ -109,33 +109,33 @@ def run(self, parameter_space, tuning_options):
                 # is served from the cache beyond the first timel. That is, when the
                 # configuration is already counted towards the unique_results.
                 if key in self.visited_results:
-                    result = util.disable_benchmark_timings(result)
+                    result = util.copy_without_benchmark_timings(result)
                 else:
                     # configuration is evaluated for the first time, print to the console
                     util.print_config_output(tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units)
                     self.visited_results.add(key)
 
-                # Simulate the evaluation of this configuration
-                tuning_options.budget.add_evaluations(1)
-                tuning_options.budget.add_time(milliseconds=result["compile_time"])
-                tuning_options.budget.add_time(milliseconds=result["verification_time"])
-                tuning_options.budget.add_time(milliseconds=result["benchmark_time"])
-
                 try:
                     self.total_simulated_time += result["compile_time"] + result["verification_time"] + result["benchmark_time"]
                 except KeyError:
                     raise RuntimeError(
                         "Cannot use simulation mode with a time limit on a cache file that does not have full compile, verification, and benchmark timings on all configurations"
                     )
 
+                # Simulate the evaluation of this configuration
+                tuning_options.budget.add_evaluations(1)
+                tuning_options.budget.add_time(milliseconds=result["compile_time"])
+                tuning_options.budget.add_time(milliseconds=result["verification_time"])
+                tuning_options.budget.add_time(milliseconds=result["benchmark_time"])
+
                 results.append(result)
                 continue
 
             # if the configuration is not in the cache and not within restrictions, simulate an InvalidConfig with warning
             params_dict = dict(zip(tuning_options['tune_params'].keys(), element))
             check = util.check_restrictions(tuning_options.restrictions, params_dict, True)
             if not check:
-                result = util.disable_benchmark_timings(params_dict) # Set timings to zero
+                result = util.copy_without_benchmark_timings(params_dict) # Set timings to zero
                 result[tuning_options.objective] = util.InvalidConfig()
                 results.append(result)
                 warn(f"Configuration {element} not in cache, does not pass restrictions. Will be treated as an InvalidConfig, but make sure you are evaluating the correct cache file.")
@@ -159,8 +159,8 @@ def run(self, parameter_space, tuning_options):
             for result in results:
                 if result:
                     # Time must be in ms
-                    result["strategy_time"] = strategy_time / num_valid_results
-                    result["framework_time"] = framework_time / num_valid_results
+                    result["strategy_time"] = 1000 * strategy_time / num_valid_results
+                    result["framework_time"] = 1000 * framework_time / num_valid_results
 
 
         return results

kernel_tuner/strategies/common.py

@@ -71,7 +71,6 @@ def __init__(
         scaling=False,
         snap=True,
         return_invalid=False,
-        return_raw=None,
         invalid_value=sys.float_info.max,
     ):
         """An abstract method to handle evaluation of configurations.
@@ -207,7 +206,7 @@ def eval_all(self, xs, check_restrictions=True):
             else:
                 # this is not a valid configuration, replace with float max if needed
                 if not self.return_invalid:
-                    return_value = sys.float_info.max
+                    return_value = self.invalid_return_value
 
             # include raw data in return if requested
             return_values.append(return_value)