Merge branch 'main' into ilgar

Author: jan-janssen

.github/workflows/pipeline.yml

@@ -3,7 +3,11 @@ name: Pipeline
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
   pull_request:
+    paths-ignore:
+      - 'docs/**'
 
 jobs:
   black:

README.md

@@ -26,7 +26,7 @@ executorlib is the lightest path to take *existing* Python functions and scale t
 library [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects) you already
 know, rather than asking you to adopt a new data, actor, or workflow model.
 
-| | executorlib | [Concurrent futures](https://docs.python.org/3/library/concurrent.futures.html) | [Dask](https://www.dask.org) | [Parsl](https://parsl-project.org) | [Ray](https://www.ray.io) |
+| | executorlib | [Futures](https://docs.python.org/3/library/concurrent.futures.html) | [Dask](https://www.dask.org) | [Parsl](https://parsl-project.org) | [Ray](https://www.ray.io) |
 |---|---|---|---|---|---|
 | Drop-in `Executor` API | ✅ | ✅ | ⚠️ | ⚠️  | ❌ |
 | Per-call resource assignment | ✅ | ❌ | ⚠️ | ✅ | ✅ |

docs/comparison.md

@@ -11,7 +11,7 @@ alternative is the better choice.
 
 ## At a glance
 
-| | executorlib | Concurrent futures | Dask | Parsl | Ray |
+| | executorlib | Futures | Dask | Parsl | Ray |
 |---|---|---|---|---|---|
 | Drop-in `Executor` API | ✅ | ✅ | ⚠️ | ⚠️  | ❌ |
 | Per-call resource assignment | ✅ | ❌ | ⚠️ | ✅ | ✅ |
@@ -22,7 +22,7 @@ alternative is the better choice.
 
 ✅ first-class · ⚠️ possible via an add-on or extra configuration · ❌ not supported.
 
-## [Concurrent futures](https://docs.python.org/3/library/concurrent.futures.html)
+## [Futures](https://docs.python.org/3/library/concurrent.futures.html)
 
 The [`concurrent.futures`](https://docs.python.org/3/library/concurrent.futures.html) module is where most parallel
 Python starts: `ProcessPoolExecutor` and `ThreadPoolExecutor` run functions in parallel on a single machine. executorlib

notebooks/2-hpc-cluster.ipynb

@@ -5,10 +5,12 @@
 _logged_failed_ids: set = set()
 
 
-def batched_futures(lst: list[Future], skip_lst: list[list], n: int) -> list[list]:
+def batched_futures(
+    lst: list[Future], nested_skip_lst: list[Future[list]], n: int
+) -> list[list]:
     """
     Batch n completed future objects. If the number of completed futures is smaller than n and the end of the batch is
-    not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_lst
+    not reached yet, then an empty list is returned. If n future objects are done, which are not included in the skip_set
     then they are returned as batch.
 
     Futures that completed with an EXCEPTION (e.g. a labeling job that failed on a degenerate config, or a dead worker)
@@ -21,7 +23,7 @@ def batched_futures(lst: list[Future], skip_lst: list[list], n: int) -> list[lis
 
     Args:
         lst (list): list of all future objects
-        skip_lst (list): list of previous batches of future objects
+        nested_skip_lst (list): nest list of individual results already assigned to previous batches
         n (int): batch size
 
     Returns:

src/executorlib/standalone/batched.py

@@ -2,7 +2,7 @@
 
 
 class TestResults(unittest.TestCase):
-    def test_result(self):
+    def test_timing_benchmarks(self):
         with open("timing.log") as f:
             content = f.readlines()
         timing_dict = {l.split()[0]: float(l.split()[1]) for l in content}

tests/benchmark/test_results.py

@@ -110,7 +110,7 @@ def test_get_future_from_cache(self):
                 cache_key="error",
             )
 
-    def test_empty(self):
+    def test_submit_single_task_and_verify_cache(self):
         with TestClusterExecutor(cache_directory="rather_this_dir") as exe:
             cloudpickle_register(ind=1)
             future = exe.submit(foo,1)

tests/unit/executor/test_api.py

@@ -52,14 +52,14 @@ def raise_error(parameter):
 
 
 class TestExecutorWithDependencies(unittest.TestCase):
-    def test_executor(self):
+    def test_future_chaining_resolves_dependency(self):
         with SingleNodeExecutor(max_cores=1) as exe:
             cloudpickle_register(ind=1)
             future_1 = exe.submit(add_function, 1, parameter_2=2)
             future_2 = exe.submit(add_function, 1, parameter_2=future_1)
             self.assertEqual(future_2.result(), 4)
 
-    def test_executor_no_wait(self):
+    def test_shutdown_no_wait_still_resolves_futures(self):
         exe = SingleNodeExecutor(max_cores=1)
         cloudpickle_register(ind=1)
         future_1 = exe.submit(add_function, 1, parameter_2=2)

tests/unit/executor/test_single_dependencies.py

@@ -33,7 +33,7 @@ def mpi_funct_sleep(i):
 
 
 class TestExecutorBackend(unittest.TestCase):
-    def test_meta_executor_serial(self):
+    def test_block_allocation_serial_two_cores(self):
         with SingleNodeExecutor(max_cores=2, block_allocation=True) as exe:
             cloudpickle_register(ind=1)
             fs_1 = exe.submit(calc, 1)
@@ -43,7 +43,7 @@ def test_meta_executor_serial(self):
             self.assertTrue(fs_1.done())
             self.assertTrue(fs_2.done())
 
-    def test_meta_executor_single(self):
+    def test_block_allocation_serial_one_core(self):
         with SingleNodeExecutor(max_cores=1, block_allocation=True) as exe:
             cloudpickle_register(ind=1)
             fs_1 = exe.submit(calc, 1)
@@ -62,7 +62,7 @@ def test_oversubscribe(self):
     @unittest.skipIf(
         skip_mpi4py_test, "mpi4py is not installed, so the mpi4py tests are skipped."
     )
-    def test_meta_executor_parallel(self):
+    def test_block_allocation_mpi_two_cores(self):
         with SingleNodeExecutor(
             max_workers=2,
             resource_dict={"cores": 2},
@@ -73,7 +73,7 @@ def test_meta_executor_parallel(self):
             self.assertEqual(fs_1.result(), [(1, 2, 0), (1, 2, 1)])
             self.assertTrue(fs_1.done())
 
-    def test_errors(self):
+    def test_invalid_constructor_arguments_raise_errors(self):
         with self.assertRaises(TypeError):
             SingleNodeExecutor(
                 max_cores=1,
@@ -88,7 +88,7 @@ def tearDown(self):
     @unittest.skipIf(
         skip_mpi4py_test, "mpi4py is not installed, so the mpi4py tests are skipped."
     )
-    def test_meta_executor_parallel_cache(self):
+    def test_block_allocation_mpi_two_cores_with_cache(self):
         with SingleNodeExecutor(
             max_workers=2,
             resource_dict={"cores": 2},

tests/unit/executor/test_single_mpi.py

@@ -29,7 +29,7 @@ def exit_funct():
 
 
 class TestExecutorBackend(unittest.TestCase):
-    def test_meta_executor_serial_with_dependencies(self):
+    def test_no_block_allocation_disable_dependencies(self):
         with SingleNodeExecutor(
             max_cores=2,
             block_allocation=False,
@@ -43,7 +43,7 @@ def test_meta_executor_serial_with_dependencies(self):
             self.assertTrue(fs_1.done())
             self.assertTrue(fs_2.done())
 
-    def test_meta_executor_serial_without_dependencies(self):
+    def test_no_block_allocation_with_dependencies(self):
         with SingleNodeExecutor(
             max_cores=2,
             block_allocation=False,
@@ -57,7 +57,7 @@ def test_meta_executor_serial_without_dependencies(self):
             self.assertTrue(fs_1.done())
             self.assertTrue(fs_2.done())
 
-    def test_meta_executor_single(self):
+    def test_no_block_allocation_single_core(self):
         with SingleNodeExecutor(
             max_cores=1,
             block_allocation=False,
@@ -70,7 +70,7 @@ def test_meta_executor_single(self):
             self.assertTrue(fs_1.done())
             self.assertTrue(fs_2.done())
 
-    def test_errors(self):
+    def test_invalid_constructor_arguments_raise_errors(self):
         with self.assertRaises(TypeError):
             SingleNodeExecutor(
                 max_cores=1,