Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
ef477d0
proposed fix for https://github.com/snakemake/snakemake-executor-plug…
Mar 6, 2026
8d033ce
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters Mar 9, 2026
ed1e50d
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters Mar 9, 2026
677c66c
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters Mar 18, 2026
f95510c
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters Mar 31, 2026
4bcb0af
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters May 18, 2026
1770a76
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters May 22, 2026
ba6f127
Merge branch 'main' of github.com:snakemake/snakemake-executor-plugin…
cmeesters Jun 8, 2026
b1df606
feat: first non-tested, intermediate version
cmeesters Jun 17, 2026
10e947d
fix: relaxing system requirements for local cluster OS
cmeesters Jun 18, 2026
f4a13a8
fix: updated checkout versions
cmeesters Jun 19, 2026
7092cdc
fix: typo
cmeesters Jun 19, 2026
6c13ec8
refactor: updates
cmeesters Jun 19, 2026
40fc0de
fix: formatting
cmeesters Jun 19, 2026
65add9c
refactor: sould resemble snakemake's main release script
cmeesters Jun 19, 2026
8ecf620
fix: poetry -> pixi
cmeesters Jun 19, 2026
f01b42c
refactor: release please versions
cmeesters Jun 19, 2026
0bc77b9
fix: the setup experiment does not belong to the repo - has been acci…
cmeesters Jun 19, 2026
0e25162
fix: not applying E501 exception to scontrol parsing test any longer
cmeesters Jun 19, 2026
a881188
fix: deleted unnecessarry pixi.toml file
cmeesters Jun 22, 2026
7e246ed
fix: working pyproject toml version
cmeesters Jun 23, 2026
f27d7e2
fix: restored functionality of release-please - hopefully
cmeesters Jun 23, 2026
3521d66
fix: attempt to run pixi install correctly
cmeesters Jun 23, 2026
da3151a
fix: added publish and build envs and check
cmeesters Jun 23, 2026
3f6aa26
fix: cleanup (removed empty task)
cmeesters Jun 23, 2026
02f034d
fix: added minor versions for actions - no need to update for every fix
cmeesters Jun 23, 2026
db1f47f
fix: need to specify dev env? + explicit install step
cmeesters Jun 24, 2026
0c22b8b
fix: install package in non-editable mode for plugin discovery
Copilot Jun 24, 2026
f4fa352
fix: another attempt
cmeesters Jun 24, 2026
17f82cd
fix: attempting explicit install step
cmeesters Jun 24, 2026
223a06c
fix: added entry point
cmeesters Jun 24, 2026
cc5a286
fix: attempting editable install
cmeesters Jun 24, 2026
6e17178
fix: syntax breaking typo
cmeesters Jun 24, 2026
8d1ff56
fix: unified env name scheme
cmeesters Jun 24, 2026
82860f1
fix: trying to get envs straight
cmeesters Jun 24, 2026
d5fbd53
fix: again ...
cmeesters Jun 24, 2026
60c79cd
fix: gitignore added some auto generated files
cmeesters Jun 25, 2026
5cf1020
fix: pyproject.toml restructured
cmeesters Jun 25, 2026
0312c68
fix: removed on-fly comments
cmeesters Jun 25, 2026
1c1fa3a
fix: attempt to run tests with more verbosity
cmeesters Jun 25, 2026
ef052c0
fix: trying alternate, more verbose test
cmeesters Jun 25, 2026
424018f
fix: reverted change of release-please script - this PR should focus …
cmeesters Jun 25, 2026
0cb4978
fix: removed entry point which is supposed to guarantee recognition -…
cmeesters Jun 26, 2026
4cb00a9
fix: removed unnecessary exception
cmeesters Jun 26, 2026
db8a7a4
fix: trying without pixi.build workspace spec
cmeesters Jun 29, 2026
6b58bce
fix: deleted black dependency (leftover)
cmeesters Jun 29, 2026
f33ab23
fix: added snakemake as depencdency to dev env
cmeesters Jul 1, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 20 additions & 33 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,41 +11,29 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out the code
uses: actions/checkout@v4
uses: actions/checkout@v6

- uses: actions/setup-python@v5
- name: Setup pixi
uses: prefix-dev/setup-pixi@v0
Comment thread
coderabbitai[bot] marked this conversation as resolved.
with:
python-version: "3.11"

- name: Install poetry
run: pip install poetry

- name: Install dependencies
run: |
poetry install --sync
environments: dev

- name: Check formatting
run: poetry run black --check .
run: pixi run --environment dev format --check .

linting:
runs-on: ubuntu-latest
steps:
- name: Check out the code
uses: actions/checkout@v4
uses: actions/checkout@v6

- uses: actions/setup-python@v5
- name: Setup pixi
uses: prefix-dev/setup-pixi@v0
with:
python-version: "3.11"

- name: Install poetry
run: pip install poetry

- name: Install dependencies
run: |
poetry install --sync
environments: dev

- name: Check code
run: poetry run flake8
run: pixi run --environment dev lint

testing:
runs-on: ubuntu-latest
Expand All @@ -58,23 +46,22 @@ jobs:
- "8888:3306"
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6

- uses: koesterlab/setup-slurm-action@v1

- uses: actions/setup-python@v5
- name: Setup pixi
uses: prefix-dev/setup-pixi@v0
with:
python-version: "3.11"

- name: Install poetry
run: pip install poetry
environments: dev

- name: Install dependencies
run: |
poetry install --sync
- name: Install plugin package
run: pixi run --environment dev python -m pip install -e .

- name: Run pytest
run: poetry run coverage run -m pytest tests/ -sv --tb=short --disable-warnings
run: |
pixi run --environment dev \
python -m pytest -vv --capture=no tests

- name: Run Coverage
run: poetry run coverage report -m
run: pixi run --environment dev coverage-report
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,6 @@ cython_debug/
#.idea/

.aider*
# pixi environments
.pixi/*
!.pixi/config.toml
128 changes: 96 additions & 32 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,49 +1,113 @@
[tool.poetry]
[build-system]
requires = ["setuptools>=64"]
build-backend = "setuptools.build_meta"


[project]
name = "snakemake-executor-plugin-slurm"
version = "2.7.1"
description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
readme = "README.md"
requires-python = ">=3.11,<4"
license = {text = "MIT"}
authors = [
"Christian Meesters <meesters@uni-mainz.de>",
"David Lähnemann <david.laehnemann@dkfz-heidelberg.de>",
"Johannes Koester <johannes.koester@uni-due.de>",
{name = "Christian Meesters", email = "meesters@uni-mainz.de"},
{name = "David Lähnemann", email = "david.laehnemann@dkfz-heidelberg.de"},
{name = "Johannes Koester", email = "johannes.koester@uni-due.de"},
]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Science/Research",
"Natural Language :: English",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering",
]
readme = "README.md"
license = "MIT"
repository = "https://github.com/snakemake/snakemake-executor-plugin-slurm"
documentation = "https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html"
keywords = ["snakemake", "plugin", "executor", "cluster", "slurm"]
dependencies = [
"snakemake-interface-common>=1.23.0,<2",
"snakemake-interface-executor-plugins>=9.4.0,<10",
"snakemake-executor-plugin-slurm-jobstep>=0.6.0,<1",
"pandas>=2.3.3,<4",
"numpy>=2.4.5,<3",
"throttler>=1.2.2,<2",
"pyyaml>=6.0.3,<7",
]

[tool.poetry.dependencies]
python = "^3.11"
snakemake-interface-common = ">=1.23.0,<2"
snakemake-interface-executor-plugins = ">=9.4.0,<10"
snakemake-executor-plugin-slurm-jobstep = ">=0.6.0,<1"
pandas = ">=2.3.3,<4"
numpy = ">=2.4.5,<3"
throttler = ">=1.2.3,<2"
pyyaml = ">=6.0.3,<7"

[tool.poetry.group.dev.dependencies]
black = "==26.3.1"
flake8 = ">=6.1.0,<7"
coverage = ">=7.14.0,<8"
pytest = ">=9.0.3,<10"
snakemake = ">=9.21.0,<10"
pandas = ">=2.3.3,<4"
[project.urls]
Repository = "https://github.com/snakemake/snakemake-executor-plugin-slurm"
Documentation = "https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html"

[tool.poetry.scripts]
[project.scripts]
generate-slurm-partition-config = "snakemake_executor_plugin_slurm.cli:main"

[tool.codespell]
skip = ".git,*.pdf,*.svg,versioneer.py,*.css,test_*"
check-hidden = true

[tool.coverage.run]
omit = [".*", "*/site-packages/*", "Snakefile"]

[tool.black]
line-length = 88
extend-exclude = 'tests/test_scontrol_parsing\.py'
[tool.ruff]
target-version = "py311"
extend-exclude = ["tests/test_scontrol_parsing.py"]

[tool.pytest.ini_options]
python_files = ["test_*.py", "tests.py"]

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.pixi]

# Workspace table (required for preview features)
[tool.pixi.workspace]
#preview = ["pixi-build"]
channels = ["conda-forge", "bioconda"]
platforms = ["linux-64"]

# Conda-only dependencies (must be a table)
[tool.pixi.dependencies]
python = ">=3.11,<3.14"
pip = ">=24,<26"
packaging = ">=24,<26"
Comment on lines +71 to +73

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do you need these dependencies?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, avoid conda dependencies and remove the entire section.
The python version should instead be defined via the requires-python key under [project].

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will try. Might have to do with my mingling with conda and pixi. Meanwhile, I have pixi-only env, too.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see the other comment: Locally, not within the CI, I still seem to need pip to install in working conda envs. Yes, pixi allows for different envs. If we want to rely on pixi, only, we need to make a few adjustments, e.g. setting the pixi cache - which seems(!) impossible in the pyproject.toml (so, such settings cannot be shipped, if I am correct?).


# Pip-only dependencies (install the plugin from local path)
[tool.pixi.pypi-dependencies]
snakemake-executor-plugin-slurm = { path = "." }

[tool.pixi.environments]
dev = { features = ["test", "style"] }
quality = { features = ["style"] }
publish = { features = ["build"] }

[tool.pixi.feature.test.pypi-dependencies]
pytest = ">=9.0.3,<10"
coverage = ">=7.14.0,<8"
snakemake = ">=9.21.0,<10"

[tool.setuptools.packages.find]
include = ["snakemake_executor_plugin_slurm*"]
# Exclude the `pr_logo` directory (and any sub‑files) from the packaged source
exclude = ["pr_logo*"]
Comment on lines +89 to +92

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is setuptools still needed now what you have migrated to pixi?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm, I tried without and got errors. Again, might be a leftover. Will test.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, I remember: I tried installing locally into a conda env with pixi and failed. So, I switched to pip. Keeping the spec in the toml lets me do that. It is not strictly necessary for pixi in the CI, yet for mixing Conda and pixi it is convenient.

I am open to alternatives: As of now, I am still new to pixi (and conda, infact).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am making a statement out of ignorance and because I failed so far. So, is this statement correct: If we want pixi-only dependency resolvement for Snakemake. Snakemake and its plugings need to share a common environment defined in all pyproject.tomls?


[dependency-groups]
dev = ["snakemake"]

[tool.pixi.feature.test.tasks]
test = {cmd = "pytest"}
coverage-report = {cmd = "coverage run -m pytest -vv --capture=no && coverage report -m"}

[tool.pixi.feature.style.pypi-dependencies]
ruff = ">=0.10.0,<0.11"

[tool.pixi.feature.style.tasks]
format = {cmd = "ruff format"}
lint = {cmd = "ruff check ."}

[tool.pixi.feature.build.pypi-dependencies]
build = "*"
twine = "*"

[tool.pixi.feature.build.tasks]
build-check = {cmd = "python -m build && twine check dist/*"}
9 changes: 0 additions & 9 deletions setup.cfg

This file was deleted.

9 changes: 3 additions & 6 deletions snakemake_executor_plugin_slurm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -510,8 +510,7 @@ def __post_init__(self, test_mode: bool = False):
"the array_limit setting to enable array job submission."
)
raise WorkflowError(
"Array job submission is effectively disabled due to "
"low array_limit."
"Array job submission is effectively disabled due to low array_limit."
)
self.slurm_logdir = _select_logdir(self.workflow)
# Check the environment variable "SNAKEMAKE_SLURM_PARTITIONS",
Expand Down Expand Up @@ -675,10 +674,8 @@ def run_jobs(self, jobs: List[JobExecutorInterface]):
"all" in self.array_jobs or rule_name in self.array_jobs
)
# TODO: use more sensible logging information, once finished
self.logger.debug(
f"Running jobs for rule: {rule_name}, " f"{same_rule_jobs}"
)
self.logger.debug("Current array job settings: " f"{self.array_jobs}")
self.logger.debug(f"Running jobs for rule: {rule_name}, {same_rule_jobs}")
self.logger.debug(f"Current array job settings: {self.array_jobs}")

if array_selected_for_rule:
dag = getattr(self.workflow, "dag", None)
Expand Down
6 changes: 3 additions & 3 deletions tests/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,9 @@ def test_simple_workflow(self, tmp_path):
report_found = True
report_path = os.path.join(expected_path, fname)
# Verify it's not empty
assert (
os.stat(report_path).st_size > 0
), f"Efficiency report {report_path} is empty"
assert os.stat(report_path).st_size > 0, (
f"Efficiency report {report_path} is empty"
)
break
assert report_found, "Efficiency report file not found"

Expand Down
2 changes: 1 addition & 1 deletion upgrade_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def sensible_range(ver):
if not m:
return None
major, minor, patch = map(int, m.groups())
return f">={major}.{minor}.{patch},<{major+1}"
return f">={major}.{minor}.{patch},<{major + 1}"

print("Upgrading pyproject.toml dependencies to sensible ranges...")
print(f"(dry-run mode: {dry_run})\n")
Expand Down
Loading