Merge branch 'main' into fast_batched

Author: jan-janssen

.ci_support/environment-docs.yml

@@ -2,8 +2,8 @@ channels:
 - conda-forge
 dependencies:
 - flux-core
-- hatchling
-- hatch-vcs
+- hatchling =1.30.1
+- hatch-vcs =0.5.0
 - myst-parser
 - nbsphinx
 - numpy

.ci_support/environment-integration.yml

@@ -12,13 +12,13 @@ dependencies:
 - flux-core =0.81.0
 - flux-pmix =0.6.0
 - gpaw =25.7.0
-- hatchling =1.29.0
+- hatchling =1.30.1
 - hatch-vcs =0.5.0
 - ipython =9.9.0
 - matplotlib =3.10.8
 - networkx =3.6.1
 - pydantic =2.13.4
 - pygraphviz =1.14
-- pysqa =0.4.1
+- pysqa =0.4.2
 - pyzmq =27.1.0
 - qe =7.5

.ci_support/environment-mini.yml

@@ -4,6 +4,6 @@ dependencies:
 - python
 - numpy =2.4.2
 - cloudpickle =3.1.0
-- hatchling =1.29.0
+- hatchling =1.30.1
 - hatch-vcs =0.5.0
 - pyzmq =27.1.0

.ci_support/environment-mpich.yml

@@ -6,12 +6,12 @@ dependencies:
 - mpich
 - cloudpickle =3.1.2
 - h5py =3.16.0
-- hatchling =1.29.0
+- hatchling =1.30.1
 - hatch-vcs =0.5.0
 - ipython =9.9.0
 - mpi4py =4.1.2
 - networkx =3.6.1
 - pydantic =2.13.4
 - pygraphviz =1.14
-- pysqa =0.4.1
+- pysqa =0.4.2
 - pyzmq =27.1.0

.ci_support/environment-openmpi.yml

@@ -6,12 +6,12 @@ dependencies:
 - openmpi
 - cloudpickle =3.1.2
 - h5py =3.16.0
-- hatchling =1.29.0
+- hatchling =1.30.1
 - hatch-vcs =0.5.0
 - ipython =9.9.0
 - mpi4py =4.1.2
 - networkx =3.6.1
 - pydantic =2.13.4
 - pygraphviz =1.14
-- pysqa =0.4.1
+- pysqa =0.4.2
 - pyzmq =27.1.0

.ci_support/environment-win.yml

@@ -6,7 +6,7 @@ dependencies:
 - msmpi
 - cloudpickle =3.1.2
 - h5py =3.16.0
-- hatchling =1.29.0
+- hatchling =1.30.1
 - hatch-vcs =0.5.0
 - ipython =9.9.0
 - mpi4py =4.1.2

.github/workflows/pipeline.yml

@@ -3,7 +3,11 @@ name: Pipeline
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
   pull_request:
+    paths-ignore:
+      - 'docs/**'
 
 jobs:
   black:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.14
+    rev: v0.15.16
     hooks:
       - id: ruff
         name: ruff lint

README.md

@@ -19,6 +19,26 @@ Up-scale python functions for high performance computing (HPC) with executorlib.
   machine learning pipelines and simulation workflows executorlib provides optional caching of intermediate results for 
   iterative development in interactive environments like jupyter notebooks.
 
+## Why executorlib?
+executorlib is the lightest path to take *existing* Python functions and scale them across high performance computing
+(HPC) nodes — with per-function-call resource control and native [SLURM](https://slurm.schedmd.com) and
+[flux](http://flux-framework.org) integration — without rewriting your code into a new paradigm. It extends the standard
+library [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects) you already
+know, rather than asking you to adopt a new data, actor, or workflow model.
+
+| | executorlib | [Futures](https://docs.python.org/3/library/concurrent.futures.html) | [Dask](https://www.dask.org) | [Parsl](https://parsl-project.org) | [Ray](https://www.ray.io) |
+|---|---|---|---|---|---|
+| Drop-in `Executor` API | ✅ | ✅ | ⚠️ | ⚠️  | ❌ |
+| Per-call resource assignment | ✅ | ❌ | ⚠️ | ✅ | ✅ |
+| Native HPC scheduler (SLURM/flux) | ✅ | ❌ | ⚠️ | ✅ | ⚠️ |
+| MPI-parallel functions | ✅ | ❌ | ⚠️ | ⚠️ | ⚠️ |
+| Caching of results | ✅ | ❌ | ⚠️ | ✅ | ❌ |
+| Setup / learning overhead | Low | Very low | Medium | Medium | Medium |
+
+✅ first-class · ⚠️ possible via add-on/config · ❌ not supported. See the full
+[comparison: when to use which](https://executorlib.readthedocs.io/en/latest/comparison.html) for honest guidance on
+when another tool is the better fit.
+
 ## Examples
 The Python standard library provides the [Executor interface](https://docs.python.org/3/library/concurrent.futures.html#executor-objects)
 with the [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor) and the 
@@ -130,6 +150,13 @@ as hierarchical job scheduler within the allocations.
   * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/installation.html#hpc-job-executor)
   * [Visualisation](https://executorlib.readthedocs.io/en/latest/installation.html#visualisation)
   * [For Developers](https://executorlib.readthedocs.io/en/latest/installation.html#for-developers)
+* [Comparison](https://executorlib.readthedocs.io/en/latest/comparison.html#comparison)
+  * [At a glace](https://executorlib.readthedocs.io/en/latest/comparison.html#at-a-glance)
+  * [Concurrent futures](https://executorlib.readthedocs.io/en/latest/comparison.html#concurrent-futures-the-python-standard-library)
+  * [Dask](https://executorlib.readthedocs.io/en/latest/comparison.html#dask)
+  * [Parsl](https://executorlib.readthedocs.io/en/latest/comparison.html#parsl)
+  * [Ray](https://executorlib.readthedocs.io/en/latest/comparison.html#ray)
+  * [Choose executorlib when](https://executorlib.readthedocs.io/en/latest/comparison.html#choose-executorlib-when)
 * [Single Node Executor](https://executorlib.readthedocs.io/en/latest/1-single-node.html)
   * [Basic Functionality](https://executorlib.readthedocs.io/en/latest/1-single-node.html#basic-functionality)
   * [Parallel Functions](https://executorlib.readthedocs.io/en/latest/1-single-node.html#parallel-functions)

binder/environment.yml

@@ -7,13 +7,13 @@ dependencies:
 - cloudpickle =3.1.2
 - flux-core =0.81.0
 - flux-pmix =0.6.0
-- hatchling =1.29.0
+- hatchling =1.30.1
 - hatch-vcs =0.5.0
 - h5py =3.16.0
 - ipython =9.9.0
 - matplotlib =3.10.0
 - mpi4py =4.1.2
 - networkx =3.6.1
 - pygraphviz =1.14
-- pysqa =0.4.1
+- pysqa =0.4.2
 - pyzmq =27.1.0