Merge pull request #358 from maric-a-b/multiobjective_optimization

Multiobjective optimization

Author: benvanwerkhoven

.gitignore

@@ -31,6 +31,7 @@ temp_*.*
 .DS_Store
 .python-version
 .nox
+.venv
 
 ### Visual Studio Code ###
 !.vscode/settings.json

kernel_tuner/__init__.py

@@ -1,6 +1,6 @@
 from importlib.metadata import version
 
-from kernel_tuner.interface import run_kernel, tune_kernel, tune_kernel_T1
+from kernel_tuner.interface import run_kernel, tune_kernel, tune_kernel_T1, tune_cache
 
 __version__ = version(__package__)
 
@@ -9,6 +9,7 @@
     "run_kernel",
     "store_results",
     "tune_kernel",
+    "tune_cache",
     "tune_kernel_T1",
     "__version__",
 ]

kernel_tuner/core.py

@@ -506,11 +506,14 @@ def benchmark(self, func, gpu_args, instance, verbose, objective, skip_nvml_sett
                     print(
                         f"skipping config {util.get_instance_string(instance.params)} reason: too many resources requested for launch"
                     )
-                result[objective] = util.RuntimeFailedConfig()
+                result['__error__'] = util.RuntimeFailedConfig()
             else:
                 logging.debug("benchmark encountered runtime failure: " + str(e))
                 print("Error while benchmarking:", instance.name)
                 raise e
+
+        assert util.check_result_type(result), "The error in a result MUST be an actual error."
+
         return result
 
     def check_kernel_output(
@@ -600,7 +603,7 @@ def compile_and_benchmark(self, kernel_source, gpu_args, params, kernel_options,
 
         instance = self.create_kernel_instance(kernel_source, kernel_options, params, verbose)
         if isinstance(instance, util.ErrorConfig):
-            result[to.objective] = util.InvalidConfig()
+            result['__error__'] = util.InvalidConfig()
         else:
             # Preprocess the argument list. This is required to deal with `MixedPrecisionArray`s
             gpu_args = _preprocess_gpu_arguments(gpu_args, params)
@@ -610,7 +613,7 @@ def compile_and_benchmark(self, kernel_source, gpu_args, params, kernel_options,
                 start_compilation = time.perf_counter()
                 func = self.compile_kernel(instance, verbose)
                 if not func:
-                    result[to.objective] = util.CompilationFailedConfig()
+                    result['__error__'] = util.CompilationFailedConfig()
                 else:
                     # add shared memory arguments to compiled module
                     if kernel_options.smem_args is not None:
@@ -655,6 +658,8 @@ def compile_and_benchmark(self, kernel_source, gpu_args, params, kernel_options,
         result["verification_time"] = last_verification_time or 0
         result["benchmark_time"] = last_benchmark_time or 0
 
+        assert util.check_result_type(result), "The error in a result MUST be an actual error."
+
         return result
 
     def compile_kernel(self, instance, verbose):

kernel_tuner/file_utils.py

@@ -20,7 +20,7 @@ def input_file_schema():
 
     :returns: the current version of the T1 schemas and the JSON string of the schema
     :rtype: string, string
-    """    
+    """
     current_version = "1.0.0"
     input_file = schema_dir.joinpath(f"T1/{current_version}/input-schema.json")
     with input_file.open() as fh:
@@ -30,9 +30,9 @@ def input_file_schema():
 def get_input_file(filepath: Path, validate=True) -> dict[str, any]:
     """Load the T1 input file from the given path, validates it and returns contents if valid.
 
-    :param filepath: Path to the input file to load. 
-    :returns: the contents of the file if valid. 
-    """    
+    :param filepath: Path to the input file to load.
+    :returns: the contents of the file if valid.
+    """
     with filepath.open() as fp:
         input_file = json.load(fp)
     if validate:
@@ -57,20 +57,38 @@ def output_file_schema(target):
     return current_version, json_string
 
 
-def get_configuration_validity(objective) -> str:
+def get_configuration_validity(error) -> str:
     """Convert internal Kernel Tuner error to string."""
     errorstring: str
-    if not isinstance(objective, util.ErrorConfig):
+    if not isinstance(error, util.ErrorConfig):
         errorstring = "correct"
     else:
-        if isinstance(objective, util.CompilationFailedConfig):
+        if isinstance(error, util.CompilationFailedConfig):
             errorstring = "compile"
-        elif isinstance(objective, util.RuntimeFailedConfig):
+        elif isinstance(error, util.RuntimeFailedConfig):
             errorstring = "runtime"
-        elif isinstance(objective, util.InvalidConfig):
+        elif isinstance(error, util.InvalidConfig):
             errorstring = "constraints"
         else:
-            raise ValueError(f"Unkown objective type {type(objective)}, value {objective}")
+            raise ValueError(f"Unkown error type {type(error)}, value {error}")
+    return errorstring
+
+
+def get_configuration_validity2(result) -> str:
+    """Convert internal Kernel Tuner error to string."""
+    errorstring: str
+    if "__error__" not in result:
+        errorstring = "correct"
+    else:
+        error = result["__error__"]
+        if isinstance(error, util.CompilationFailedConfig):
+            errorstring = "compile"
+        elif isinstance(error, util.RuntimeFailedConfig):
+            errorstring = "runtime"
+        elif isinstance(error, util.InvalidConfig):
+            errorstring = "constraints"
+        else:
+            raise ValueError(f"Unkown error type {type(error)}, value {error}")
     return errorstring
 
 
@@ -103,6 +121,11 @@ def get_t4_results(results, tune_params, objective="time"):
     :type objective: string
 
     """
+    assert not isinstance(objective, (list, tuple))
+
+    if isinstance(objective, (list, tuple)) and len(objective) > 1:
+        raise ValueError("The T4 format does not support multiple objectives.")
+
     timing_keys = ["compile_time", "benchmark_time", "framework_time", "strategy_time", "verification_time"]
     not_measurement_keys = list(tune_params.keys()) + timing_keys + ["timestamp"] + ["times"]
 
@@ -129,7 +152,8 @@ def get_t4_results(results, tune_params, objective="time"):
         out["times"] = timings
 
         # encode the validity of the configuration
-        out["invalidity"] = get_configuration_validity(result[objective])
+        # out["invalidity"] = get_configuration_validity(result[objective])
+        out["invalidity"] = get_configuration_validity2(result)
 
         # Kernel Tuner does not support producing results of configs that fail the correctness check
         # therefore correctness is always 1
@@ -143,10 +167,9 @@ def get_t4_results(results, tune_params, objective="time"):
         out["measurements"] = measurements
 
         # objectives
-        # In Kernel Tuner we currently support only one objective at a time, this can be a user-defined
-        # metric that combines scores from multiple different quantities into a single value to support
-        # multi-objective tuning however.
-        out["objectives"] = [objective]
+        objectives = [objective] if isinstance(objective, str) else list(objective)
+        assert isinstance(objectives, list)
+        out["objectives"] = objectives
 
         # append to output
         output_data.append(out)
@@ -310,7 +333,7 @@ def load_module(module_name):
         spec = spec_from_file_location(module_name, file_path)
         if spec is None:
             raise ImportError(f"Could not load spec from {file_path}")
-        
+
         # create a module from the spec and execute it
         module = module_from_spec(spec)
         spec.loader.exec_module(module)
@@ -322,6 +345,6 @@ def load_module(module_name):
         module = load_module(file_path.stem)
     except ImportError:
         module = load_module(f"{file_path.parent.stem}.{file_path.stem}")
-    
+
     # return the class from the module
     return getattr(module, class_name)

kernel_tuner/interface.py

@@ -74,6 +74,8 @@
     "pyatf_strategies": "kernel_tuner.strategies.pyatf_strategies",
     "hybrid_vndx": "kernel_tuner.strategies.gen_hybrid_vndx",
     "adaptive_tabu_greywolf": "kernel_tuner.strategies.gen_adaptive_tabu_greywolf",
+    "nsga2": "kernel_tuner.strategies.pymoo_minimize",
+    "nsga3": "kernel_tuner.strategies.pymoo_minimize",
 }
 
 _STRATEGY_PARALLEL = ["brute_force", "random_sample", "diff_evo", "genetic_algorithm", "pso", "firefly_algorithm"]
@@ -460,15 +462,15 @@ def __deepcopy__(self, _):
                 """Optimization objective to sort results on, consisting of a string
             that also occurs in results as a metric or observed quantity, default 'time'.
             Please see :ref:`objectives`.""",
-                "string",
+                "str | list[str]",
             ),
         ),
         (
             "objective_higher_is_better",
             (
                 """boolean that specifies whether the objective should
             be maximized (True) or minimized (False), default False.""",
-                "bool",
+                "bool | list[bool]",
             ),
         ),
         (
@@ -498,6 +500,7 @@ def __deepcopy__(self, _):
         ),
         ("metrics", ("specifies user-defined metrics, please see :ref:`metrics`.", "dict")),
         ("simulation_mode", ("Simulate an auto-tuning search from an existing cachefile", "bool")),
+        ("seed", ("""The random seed.""", "int")),
         ("parallel", ("Set to `True` or an integer to enable parallel tuning. If set to an integer, this will be the number of parallel workers.", "int|bool")),
         (
             "observers",
@@ -621,6 +624,8 @@ def tune_kernel(
     observers=None,
     objective=None,
     objective_higher_is_better=None,
+    objectives=None,
+    seed=None,
 ):
     start_overhead_time = perf_counter()
     if log:
@@ -630,8 +635,22 @@ def tune_kernel(
 
     _check_user_input(kernel_name, kernelsource, arguments, block_size_names)
 
-    # default objective if none is specified
-    objective, objective_higher_is_better = get_objective_defaults(objective, objective_higher_is_better)
+    if objectives:
+        if isinstance(objectives, dict):
+            objective = list(objectives.keys())
+            objective_higher_is_better = list(objectives.values())
+        else:
+            raise ValueError("objectives should be a dict of (objective, higher_is_better) pairs")
+    else:
+        objective, objective_higher_is_better = get_objective_defaults(objective, objective_higher_is_better)
+        if isinstance(objective, str):
+            objective = [objective]
+        if isinstance(objective_higher_is_better, bool):
+            objective_higher_is_better = [objective_higher_is_better]
+
+    assert isinstance(objective, list)
+    assert isinstance(objective_higher_is_better, list)
+    assert len(objective) == len(objective_higher_is_better)
 
     # check for forbidden names in tune parameters
     util.check_tune_params_list(tune_params, observers, simulation_mode=simulation_mode)
@@ -663,7 +682,6 @@ def tune_kernel(
         if "searchspace_construction_options" in strategy_options:
             searchspace_construction_options = strategy_options["searchspace_construction_options"]
 
-
     # log the user inputs
     logging.debug("tune_kernel called")
     logging.debug("kernel_options: %s", util.get_config_string(kernel_options))
@@ -776,13 +794,35 @@ def preprocess_cache(filepath):
 
     # finished iterating over search space
     if results:  # checks if results is not empty
-        best_config = util.get_best_config(results, objective, objective_higher_is_better)
-        # add the best configuration to env
-        env["best_config"] = best_config
-        if not device_options.quiet:
-            units = getattr(runner, "units", None)
-            print("best performing configuration:")
-            util.print_config_output(tune_params, best_config, device_options.quiet, metrics, units)
+        if len(objective) == 1:
+            objective = objective[0]
+            objective_higher_is_better = objective_higher_is_better[0]
+            best_config = util.get_best_config(results, objective, objective_higher_is_better)
+            # add the best configuration to env
+            env['best_config'] = best_config
+            if not device_options.quiet:
+                units = getattr(runner, "units", None)
+                keys = list(tune_params.keys())
+                keys += [objective]
+                if metrics:
+                    keys += list(metrics.keys())
+                if not quiet:
+                    print(f"\nBEST PERFORMING CONFIGURATION FOR OBJECTIVE {objective}:")
+                    print(util.get_config_string(best_config, keys, units))
+        else:
+            pareto_front = util.get_pareto_results(results, objective, objective_higher_is_better)
+            # add the best configuration to env
+            env['best_config'] = pareto_front
+            if not device_options.quiet:
+                units = getattr(runner, "units", None)
+                keys = list(tune_params.keys())
+                keys += list(objective)
+                if metrics:
+                    keys += list(metrics.keys())
+                if not quiet:
+                    print(f"\nBEST PERFORMING CONFIGURATIONS FOR OBJECTIVES: {objective}:")
+                    for best_config in pareto_front:
+                        print(util.get_config_string(best_config, keys, units))
     elif not device_options.quiet:
         print("no results to report")
 
@@ -797,6 +837,28 @@ def preprocess_cache(filepath):
 
 tune_kernel.__doc__ = _tune_kernel_docstring
 
+
+def tune_cache(
+    cache_path,
+    restrictions = None,
+    **kwargs,
+):
+    cache = util.read_cache(cache_path, open_cache=False)
+    tune_args = util.infer_args_from_cache(cache)
+    _restrictions = [util.infer_restrictions_from_cache(cache)]
+
+    # Add the user provided restrictions
+    if restrictions:
+        if isinstance(restrictions, list):
+            _restrictions.extend(restrictions)
+        else:
+            raise ValueError("The restrictions must be a list()")
+
+    tune_args.update(kwargs)
+
+    return tune_kernel(**tune_args, cache=cache_path, restrictions=_restrictions, simulation_mode=True)
+
+
 _run_kernel_docstring = """Compile and run a single kernel
 
     Compiles and runs a single kernel once, given a specific instance of the kernels tuning parameters.

kernel_tuner/runners/sequential.py

@@ -5,7 +5,7 @@
 
 from kernel_tuner.core import DeviceInterface
 from kernel_tuner.runners.runner import Runner
-from kernel_tuner.util import ErrorConfig, Timer, print_config_output, process_metrics, store_cache, copy_without_benchmark_timings
+from kernel_tuner.util import ErrorConfig, Timer, print_config_output, process_metrics, store_cache, copy_without_benchmark_timings, check_result_type
 
 
 class SequentialRunner(Runner):
@@ -41,10 +41,12 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob
         self.gpu_args = self.dev.ready_argument_list(kernel_options.arguments)
 
     def get_device_info(self):
+        """ Return the backend used by this runner. """
         return self.dev
 
     def get_environment(self, tuning_options):
-        return self.dev.get_environment()
+        env = self.dev.get_environment()
+        return env
 
     def run(self, parameter_space, tuning_options):
         """Iterate through the entire parameter space using a single Python process.
@@ -101,13 +103,15 @@ def run(self, parameter_space, tuning_options):
                     result["compile_time"] + result["verification_time"] + result["benchmark_time"]
                 ) / 1000
 
+                assert check_result_type(result)
+
                 params.update(result)
 
-                if isinstance(result.get(tuning_options.objective), ErrorConfig):
+                if '__error__' in result:
                     logging.debug("kernel configuration was skipped silently due to compile or runtime failure")
 
             # only compute metrics on configs that have not errored
-            if not isinstance(params.get(tuning_options.objective), ErrorConfig):
+            if tuning_options.metrics and '__error__' not in params:
                 params = process_metrics(params, tuning_options.metrics)
 
             params["timestamp"] = str(datetime.now(timezone.utc))
@@ -119,6 +123,8 @@ def run(self, parameter_space, tuning_options):
                 # add configuration to cache
                 store_cache(x_int, params, tuning_options.cachefile, tuning_options.cache)
 
+            assert check_result_type(params)
+
             # all visited configurations are added to results to provide a trace for optimization strategies
             results.append(params)