Provide the log_obj_size option in every Executor (#716)

* Provide the log_obj_size option in every Executor

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: jan-janssen

executorlib/executor/flux.py

@@ -4,6 +4,7 @@
 from executorlib.standalone.inputcheck import (
     check_command_line_argument_lst,
     check_init_function,
+    check_log_obj_size,
     check_oversubscribe,
     check_plot_dependency_graph,
     check_pmi,
@@ -246,6 +247,7 @@ class FluxClusterExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
 
     Examples:
         ```
@@ -282,6 +284,7 @@ def __init__(
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        log_obj_size: bool = False,
     ):
         """
         The executorlib.FluxClusterExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -323,6 +326,7 @@ def __init__(
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
 
         """
         default_resource_dict: dict = {
@@ -338,6 +342,7 @@ def __init__(
         resource_dict.update(
             {k: v for k, v in default_resource_dict.items() if k not in resource_dict}
         )
+        check_log_obj_size(log_obj_size=log_obj_size)
         if not plot_dependency_graph:
             import pysqa  # noqa
 

executorlib/executor/slurm.py

@@ -3,6 +3,7 @@
 from executorlib.executor.base import BaseExecutor
 from executorlib.standalone.inputcheck import (
     check_init_function,
+    check_log_obj_size,
     check_plot_dependency_graph,
     check_refresh_rate,
     validate_number_of_cores,
@@ -58,6 +59,7 @@ class SlurmClusterExecutor(BaseExecutor):
         plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                       debugging purposes and to get an overview of the specified dependencies.
         plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+        log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
 
     Examples:
         ```
@@ -94,6 +96,7 @@ def __init__(
         refresh_rate: float = 0.01,
         plot_dependency_graph: bool = False,
         plot_dependency_graph_filename: Optional[str] = None,
+        log_obj_size: bool = False,
     ):
         """
         The executorlib.SlurmClusterExecutor leverages either the message passing interface (MPI), the SLURM workload
@@ -135,6 +138,7 @@ def __init__(
             plot_dependency_graph (bool): Plot the dependencies of multiple future objects without executing them. For
                                           debugging purposes and to get an overview of the specified dependencies.
             plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+            log_obj_size (bool): Enable debug mode which reports the size of the communicated objects.
 
         """
         default_resource_dict: dict = {
@@ -150,6 +154,7 @@ def __init__(
         resource_dict.update(
             {k: v for k, v in default_resource_dict.items() if k not in resource_dict}
         )
+        check_log_obj_size(log_obj_size=log_obj_size)
         if not plot_dependency_graph:
             import pysqa  # noqa
 

executorlib/standalone/inputcheck.py

@@ -194,7 +194,21 @@ def validate_number_of_cores(
 
 
 def check_file_exists(file_name: Optional[str]):
+    """
+    Check if file exists and raise a ValueError if it does not or file_name is None.
+    """
     if file_name is None:
         raise ValueError("file_name is not set.")
     if not os.path.exists(file_name):
         raise ValueError("file_name is not written to the file system.")
+
+
+def check_log_obj_size(log_obj_size: bool) -> None:
+    """
+    Check if log_obj_size is True and raise a ValueError if it is.
+    """
+    if log_obj_size:
+        raise ValueError(
+            "log_obj_size is not supported for the executorlib.SlurmClusterExecutor and executorlib.FluxClusterExecutor."
+            "Please use log_obj_size=False instead of log_obj_size=True."
+        )

tests/test_standalone_inputcheck.py

@@ -18,6 +18,7 @@
     check_hostname_localhost,
     check_pysqa_config_directory,
     check_file_exists,
+    check_log_obj_size,
     validate_number_of_cores,
 )
 
@@ -119,3 +120,7 @@ def test_validate_number_of_cores(self):
             ),
             int,
         )
+
+    def test_check_log_obj_size(self):
+        with self.assertRaises(ValueError):
+            check_log_obj_size(log_obj_size=True)