From dbea8886bdd5cf84b857fbfcdb122bbbb1960fea Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:04:01 +0200 Subject: [PATCH 01/10] sync pixi.toml --- copier.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copier.yaml b/copier.yaml index 64a745f..b0b4deb 100644 --- a/copier.yaml +++ b/copier.yaml @@ -31,7 +31,7 @@ _skip_if_exists: - CITATION.cff - INTERFACE.yaml - mkdocs.yaml - - pixi.toml + # Special cases - "!config/README.md" - "!tests/clio_test.py" _templates_suffix: .jinja From 8517ac6a54dc28597d844a54e58b25cd08241371 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:09:44 +0200 Subject: [PATCH 02/10] remove mkdocs chaff --- copier.yaml | 1 - template/.readthedocs.yaml | 19 ----------- template/mkdocs.yaml.jinja | 67 -------------------------------------- template/pixi.toml.jinja | 13 -------- tests/template_test.py | 5 --- 5 files changed, 105 deletions(-) delete mode 100644 template/.readthedocs.yaml delete mode 100644 template/mkdocs.yaml.jinja diff --git a/copier.yaml b/copier.yaml index b0b4deb..ad45823 100644 --- a/copier.yaml +++ b/copier.yaml @@ -30,7 +30,6 @@ _skip_if_exists: - AUTHORS - CITATION.cff - INTERFACE.yaml - - mkdocs.yaml # Special cases - "!config/README.md" - "!tests/clio_test.py" diff --git a/template/.readthedocs.yaml b/template/.readthedocs.yaml deleted file mode 100644 index 0480acd..0000000 --- a/template/.readthedocs.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details -version: 2 - -build: - os: ubuntu-22.04 - tools: - python: "3.12" - jobs: - install: - - asdf plugin add pixi - - asdf install pixi latest - - asdf global pixi latest - build: - html: - - pixi run build-docs -d $READTHEDOCS_OUTPUT/html - -mkdocs: - configuration: mkdocs.yaml diff --git a/template/mkdocs.yaml.jinja b/template/mkdocs.yaml.jinja deleted file mode 100644 index 5fd0a73..0000000 --- a/template/mkdocs.yaml.jinja +++ /dev/null @@ -1,67 +0,0 @@ -# yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json - -site_name: {{module_long_name}} -site_url: https://{{module_short_name}}.readthedocs.io/ -repo_name: {{module_short_name}} -repo_url: https://github.com/{{github_org}}/{{module_short_name}}/ -edit_uri: edit/main/docs/ -theme: - name: material - features: - - navigation.footer - - navigation.instant - - navigation.tabs - - navigation.tabs.sticky - - navigation.indexes - - search.suggest - - search.highlight - - toc.follow - palette: - # Palette toggle for automatic mode - - media: "(prefers-color-scheme)" - toggle: - icon: material/brightness-auto - name: Switch to light mode - primary: red - accent: pink - # Palette toggle for light mode - - media: "(prefers-color-scheme: light)" - scheme: default - toggle: - icon: material/weather-sunny - name: Switch to dark mode - primary: red - accent: pink - # Palette toggle for dark mode - - media: "(prefers-color-scheme: dark)" - scheme: slate - toggle: - icon: material/weather-night - name: Switch to system preference - primary: red - accent: pink -markdown_extensions: - - admonition - - pymdownx.arithmatex: - generic: true - - pymdownx.betterem: - smart_enable: all - - pymdownx.details - - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg - - pymdownx.highlight: - anchor_linenums: true - - pymdownx.snippets: - check_paths: true - restrict_base_path: true - - pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:mermaid2.fence_mermaid_custom - - toc: - anchorlink: true -plugins: - - search -copyright: Copyright © since {{ '%Y' | strftime }} {{module_short_name}} contributors ({{license}} licensed) diff --git a/template/pixi.toml.jinja b/template/pixi.toml.jinja index 4ef4bde..4e8f19b 100644 --- a/template/pixi.toml.jinja +++ b/template/pixi.toml.jinja @@ -22,16 +22,3 @@ snakemake-minimal = ">=8.29.0" [tasks] test-integration = {cmd = "pytest tests/clio_test.py"} - -[feature.docs.dependencies] -mkdocs-material = ">=9.6.7" - -[feature.docs.pypi-dependencies] -mkdocs-mermaid2-plugin = ">=1.2.1" - -[feature.docs.tasks] -serve-docs = {cmd = "mkdocs serve"} -build-docs = {cmd = "mkdocs build"} - -[environments] -docs = ["docs"] diff --git a/tests/template_test.py b/tests/template_test.py index 44b2a10..a28a6bf 100644 --- a/tests/template_test.py +++ b/tests/template_test.py @@ -35,11 +35,6 @@ def pixi_built(template_project): return template_project -def test_mkdocs_build(pixi_built): - """The template's mkdocs should build without issues.""" - assert subprocess.run("pixi run build-docs", shell=True, check=True, cwd=pixi_built) - - def test_pytest(pixi_built): """The template's tests should pass by default.""" assert subprocess.run( From 10d716c7836297fa9889f6ea1e5aacb9eb7b5be0 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:15:48 +0200 Subject: [PATCH 03/10] Integration test should use a single core --- template/tests/clio_test.py.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template/tests/clio_test.py.jinja b/template/tests/clio_test.py.jinja index fcd4971..d810e92 100644 --- a/template/tests/clio_test.py.jinja +++ b/template/tests/clio_test.py.jinja @@ -40,7 +40,7 @@ def test_standard_file_existance(module_path, file): def test_snakemake_all_failure(module_path): """The snakemake 'all' rule should return an error by default.""" process = subprocess.run( - "snakemake --cores 4", shell=True, cwd=module_path, capture_output=True + "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True ) assert "INVALID (missing locally)" in str(process.stderr) @@ -48,7 +48,7 @@ def test_snakemake_all_failure(module_path): def test_snakemake_integration_testing(module_path): """Run a light-weight test simulating someone using this module.""" assert subprocess.run( - "snakemake --use-conda --cores 4", + "snakemake --use-conda --cores 1", shell=True, check=True, cwd=module_path / "tests/integration", From faf347d0ff9b15194dfabe7972abb2ff877b2a01 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:25:27 +0200 Subject: [PATCH 04/10] lint main workflow and integration workflow --- .github/workflows/template-pr-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/template-pr-ci.yml b/.github/workflows/template-pr-ci.yml index c8aff93..7381f51 100644 --- a/.github/workflows/template-pr-ci.yml +++ b/.github/workflows/template-pr-ci.yml @@ -22,7 +22,9 @@ jobs: continue-on-error: true - name: Run snakemake linting id: linting - run: pixi run snakemake --lint + run: | + pixi run snakemake --lint --snakefile workflow/Snakefile + pixi run snakemake --lint --snakefile tests/integration/Snakefile continue-on-error: true - name: Save integration logs if: ${{ always() }} From 2f321680451dd271903eedb00ecfdc0f4321605f Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:38:55 +0200 Subject: [PATCH 05/10] tell mypy to ignore .pixi and .snakemake folders --- mypy.ini | 1 + template/mypy.ini.jinja | 1 + 2 files changed, 2 insertions(+) diff --git a/mypy.ini b/mypy.ini index d787271..c7b2d53 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] disable_error_code = import-untyped +exclude = (^|/)\.(snakemake|pixi)(/|$) diff --git a/template/mypy.ini.jinja b/template/mypy.ini.jinja index d787271..c7b2d53 100644 --- a/template/mypy.ini.jinja +++ b/template/mypy.ini.jinja @@ -1,2 +1,3 @@ [mypy] disable_error_code = import-untyped +exclude = (^|/)\.(snakemake|pixi)(/|$) From 88e7bb3a1f5940247633cf52e6e957ba1e553d05 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:43:44 +0200 Subject: [PATCH 06/10] Ensure json 2012 draft compatibility --- template/pixi.toml.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/template/pixi.toml.jinja b/template/pixi.toml.jinja index 4e8f19b..9c8a2ff 100644 --- a/template/pixi.toml.jinja +++ b/template/pixi.toml.jinja @@ -13,6 +13,7 @@ clio-tools = ">=2025.03.03" conda = ">=25.0.0" ipdb = ">=0.13.13" ipykernel = ">=6.29.5" +jsonschema = ">=4.0.0" mypy = ">=1.15.0" pytest = ">=8.3.5" python = ">=3.12" From b2925bd47f1e2ee9c35ca600f1b65f968e7bc1a5 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 21 Jul 2025 16:50:27 +0200 Subject: [PATCH 07/10] Add example of local file inclusion to main snakefile --- template/workflow/Snakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/template/workflow/Snakefile b/template/workflow/Snakefile index 3fe07e2..4424fa8 100644 --- a/template/workflow/Snakefile +++ b/template/workflow/Snakefile @@ -21,6 +21,8 @@ with open(workflow.source_path("internal/settings.yaml"), "r") as f: include: "rules/automatic.smk" include: "rules/dummy.smk" +# Add additional files to be delivered alongside the workflow here. +# e.g.: workflow.source_path("scripts/_utils.py") rule all: message: From c0eb48f644b5e8ce2dfd8782e6efb4f5eee45f1f Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:11:49 +0200 Subject: [PATCH 08/10] Ignore pixi environment folder --- template/.gitignore.jinja | 3 +++ 1 file changed, 3 insertions(+) diff --git a/template/.gitignore.jinja b/template/.gitignore.jinja index 21787f0..f03928b 100644 --- a/template/.gitignore.jinja +++ b/template/.gitignore.jinja @@ -11,6 +11,9 @@ __pycache__ *.pyc +### Environments +.pixi/ + ### Snakemake .snakemake/ gurobi.log From 4e6bb22eb2ea92d3757e6cf08364b31bc65ba266 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:37:25 +0200 Subject: [PATCH 09/10] PR improvements --- copier.yaml | 2 +- template/workflow/Snakefile | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/copier.yaml b/copier.yaml index ad45823..0f66b2a 100644 --- a/copier.yaml +++ b/copier.yaml @@ -30,7 +30,7 @@ _skip_if_exists: - AUTHORS - CITATION.cff - INTERFACE.yaml - # Special cases + # Special cases: these files should not be user-modified, even though they are in user-modifiable directories - "!config/README.md" - "!tests/clio_test.py" _templates_suffix: .jinja diff --git a/template/workflow/Snakefile b/template/workflow/Snakefile index 4424fa8..548c651 100644 --- a/template/workflow/Snakefile +++ b/template/workflow/Snakefile @@ -22,6 +22,8 @@ include: "rules/automatic.smk" include: "rules/dummy.smk" # Add additional files to be delivered alongside the workflow here. +# This is needed e.g. for python code files that are never +# explicitly used as a script or input in a snakemake rule. # e.g.: workflow.source_path("scripts/_utils.py") rule all: From 8df1660482f16b34b7e8e6bf33f8d66bcf4168b3 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:57:08 +0200 Subject: [PATCH 10/10] Add link to snakemake workflow requirements --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e93b97c..af73a96 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,8 @@ We recommend using [`pixi`](https://pixi.sh/) as your package manager. Once inst ## Features -- Standardised layout compliant with the [snakemake workflow catalogue's](https://snakemake.github.io/snakemake-workflow-catalog/#) listing requirements. +- Standardised layout compliant with the [snakemake workflow catalogue's](https://snakemake.github.io/snakemake-workflow-catalog/#) listing requirements, enabling them to be automatically included in their listings once published. Read more about those requirements [here](https://snakemake.github.io/snakemake-workflow-catalog/docs/catalog.html#standardized-usage-workflows). +- Standardised input/output structure across modules: - `resources/`: files needed for the module's processes. - `user/`: files that should be provided by users. Document them well! - `automatic/`: files that the module downloads or prepares in intermediate steps. @@ -56,7 +57,6 @@ We recommend using [`pixi`](https://pixi.sh/) as your package manager. Once inst - Continuous Integration (CI) settings, ready for [pre-commit.ci](https://pre-commit.ci/). - GitHub actions to automate chores during pull requests and releases. - Premade `pytest` setup. -- Documentation setup, ready for [Read the Docs](https://about.readthedocs.com/) or [Github Pages](https://pages.github.com/). > [!IMPORTANT] >