From 7c89485d98e024dd7853a73ea10a6e8a4683fa33 Mon Sep 17 00:00:00 2001 From: Nils Homer Date: Thu, 16 Jul 2026 00:16:27 -0400 Subject: [PATCH] feat: support snakemake 9 The executor talks to snakemake exclusively through snakemake-interface-executor-plugins, whose contract is identical across snakemake 8 and 9, so no source changes are required. This documents and tests support for both majors: - widen the dev snakemake dependency to >=8.30,<10; - add a CI test matrix over snakemake 8 and 9 (each leg resolves a different snakemake-interface-storage-plugins major via snakemake-storage-plugin-s3, so both must be exercised); - state supported versions in the docs and make the container-image storage-plugin guidance symmetric across snakemake 8 and 9. --- .github/workflows/ci_mocked_api.yml | 15 +++++++++++++-- docs/further.md | 19 ++++++++++--------- docs/intro.md | 4 +++- pyproject.toml | 6 +++++- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_mocked_api.yml b/.github/workflows/ci_mocked_api.yml index aba7865..81b2d36 100644 --- a/.github/workflows/ci_mocked_api.yml +++ b/.github/workflows/ci_mocked_api.yml @@ -70,6 +70,14 @@ jobs: testing: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Exercise both supported snakemake majors. The two legs resolve to + # different snakemake-interface-storage-plugins majors (3.x vs 4.x, via + # snakemake-storage-plugin-s3), so passing on one does not imply the other. + snakemake: ["<9", ">=9,<10"] + name: testing (snakemake ${{ matrix.snakemake }}) steps: - uses: actions/checkout@v3 @@ -80,8 +88,11 @@ jobs: - name: Install poetry run: pip install poetry - - name: Determine dependencies - run: poetry lock + - name: Constrain snakemake for this matrix leg + # A full re-resolve (not an overlay install) is required: the snakemake + # major dictates the storage-interface major, which in turn pins a + # compatible snakemake-storage-plugin-s3 version. + run: poetry add --group dev --lock "snakemake@${{ matrix.snakemake }}" - uses: actions/setup-python@v4 with: diff --git a/docs/further.md b/docs/further.md index ea11953..09d23c0 100644 --- a/docs/further.md +++ b/docs/further.md @@ -91,15 +91,16 @@ snakemake --jobs 4 \ # Container Image Requirements -The plugin does **not** auto-deploy the default storage provider to workers -(`auto_deploy_default_storage_provider=False`): workers no longer run -`pip install snakemake-storage-plugin-s3` at startup, because that pulls an -unpinned version whose newer releases require snakemake >= 9 and break -workers running snakemake 8.x. The container image used for jobs must -therefore pre-install a compatible version of the storage plugin (e.g. -`snakemake-storage-plugin-s3`) alongside snakemake itself. Image maintainers -are responsible for pinning a plugin version compatible with the snakemake -version in the image. +The plugin supports Snakemake 8 and 9 and does **not** auto-deploy the default +storage provider to workers (`auto_deploy_default_storage_provider=False`): +workers do not run `pip install snakemake-storage-plugin-s3` at startup, because +an unpinned install can pull a storage-plugin version whose +`snakemake-interface-storage-plugins` requirement does not match the Snakemake +major baked into the image, breaking the worker (Snakemake 8 needs a plugin +built against `snakemake-interface-storage-plugins` 3.x; Snakemake 9, one built +against 4.x). The container image used for jobs must therefore pre-install a +storage-plugin version compatible with its Snakemake version. Image maintainers +are responsible for pinning a compatible pair. # Scheduling Priority (Fair-Share Queues) diff --git a/docs/intro.md b/docs/intro.md index 172d511..df0688d 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -1,2 +1,4 @@ This is the Snakemake plugin for [AWS Batch](https://aws.amazon.com/batch/). This plugin -is used to distribute Snakemake jobs to AWS Batch EC2 instances. \ No newline at end of file +is used to distribute Snakemake jobs to AWS Batch EC2 instances. + +This plugin supports Snakemake 8 and 9. diff --git a/pyproject.toml b/pyproject.toml index 9626d11..3a395a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,11 @@ black = "^23.11.0" flake8 = "^6.1.0" coverage = "^7.3.2" pytest = "^7.4.3" -snakemake = "^8.30.0" +# Support both snakemake 8 and 9. The executor talks to snakemake only through +# snakemake-interface-executor-plugins (pinned in the runtime deps above), whose +# contract is identical across 8 and 9, so no source changes are needed. CI's +# testing matrix constrains this per leg to exercise both majors. +snakemake = ">=8.30,<10" [tool.coverage.run] omit = [".*", "*/site-packages/*", "Snakefile"]