Merge branch 'main' into feature-ci-cdsapi

irm-codebase · irm-codebase · commit 9a6734cecb07 · 2025-08-08T14:38:33.000+02:00
diff --git a/.github/workflows/template-pr-ci.yml b/.github/workflows/template-pr-ci.yml
@@ -28,9 +28,26 @@ jobs:
           else
             echo "CDSAPI_KEY secret is empty or not set. Skipping .cdsapirc setup."
           fi
-      - name: setup-pixi
+      - name: Setup pixi
         uses: prefix-dev/setup-pixi@v0.8.3
-      - name: Run tests
-        run: pixi run pytest tests/clio_test.py
-      - name: Run linting
-        run: pixi run snakemake --lint
+      - name: Run integration tests
+        id: tests
+        run: pixi run test-integration
+        continue-on-error: true
+      - name: Run snakemake linting
+        id: linting
+        run: |
+          pixi run snakemake --lint --snakefile workflow/Snakefile
+          pixi run snakemake --lint --snakefile tests/integration/Snakefile
+        continue-on-error: true
+      - name: Save integration logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-logs-${{ matrix.os }}
+          path: tests/integration/results/integration_test/logs
+          if-no-files-found: ignore
+          retention-days: 30
+      - name: Fail if integration or linting failed
+        if: ${{ steps.tests.outcome == 'failure' || steps.linting.outcome == 'failure' }}
+        run: exit 1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -24,7 +24,7 @@ repos:
 
   # Python file formatting
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.12
+    rev: v0.12.7
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
diff --git a/README.md b/README.md
@@ -47,7 +47,8 @@ We recommend using [`pixi`](https://pixi.sh/) as your package manager. Once inst
 
 ## Features
 
-- Standardised layout compliant with the [snakemake workflow catalogue's](https://snakemake.github.io/snakemake-workflow-catalog/#) listing requirements.
+- Standardised layout compliant with the [snakemake workflow catalogue's](https://snakemake.github.io/snakemake-workflow-catalog/#) listing requirements, enabling them to be automatically included in their listings once published. Read more about those requirements [here](https://snakemake.github.io/snakemake-workflow-catalog/docs/catalog.html#standardized-usage-workflows).
+- Standardised input/output structure across modules:
   - `resources/`: files needed for the module's processes.
     - `user/`: files that should be provided by users. Document them well!
     - `automatic/`: files that the module downloads or prepares in intermediate steps.
@@ -56,7 +57,6 @@ We recommend using [`pixi`](https://pixi.sh/) as your package manager. Once inst
   - Continuous Integration (CI) settings, ready for [pre-commit.ci](https://pre-commit.ci/).
   - GitHub actions to automate chores during pull requests and releases.
   - Premade `pytest` setup.
-- Documentation setup, ready for [Read the Docs](https://about.readthedocs.com/) or [Github Pages](https://pages.github.com/).
 
 > [!IMPORTANT]
 >
diff --git a/copier.yaml b/copier.yaml
@@ -30,8 +30,7 @@ _skip_if_exists:
   - AUTHORS
   - CITATION.cff
   - INTERFACE.yaml
-  - mkdocs.yaml
-  - pixi.toml
+  # Special cases: these files should not be user-modified, even though they are in user-modifiable directories
   - "!config/README.md"
   - "!tests/clio_test.py"
 _templates_suffix: .jinja
diff --git a/mypy.ini b/mypy.ini
@@ -1,2 +1,3 @@
 [mypy]
 disable_error_code = import-untyped
+exclude = (^|/)\.(snakemake|pixi)(/|$)
diff --git a/template/.gitignore.jinja b/template/.gitignore.jinja
@@ -11,6 +11,9 @@
 __pycache__
 *.pyc
 
+### Environments
+.pixi/
+
 ### Snakemake
 .snakemake/
 gurobi.log
diff --git a/template/.readthedocs.yaml b/template/.readthedocs.yaml
diff --git a/template/README.md.jinja b/template/README.md.jinja
@@ -23,7 +23,7 @@ pixi install --all
 For testing, simply run:
 
 ```shell
-pixi run test
+pixi run test-integration
 ```
 
 To view the documentation locally, use:
@@ -37,5 +37,5 @@ To test a minimal example of a workflow using this module:
 ```shell
 pixi shell    # activate this project's environment
 cd tests/integration/  # navigate to the integration example
-snakemake --use-conda  # run the workflow!
+snakemake --use-conda --cores 2  # run the workflow!
 ```
diff --git a/template/mkdocs.yaml.jinja b/template/mkdocs.yaml.jinja
diff --git a/template/mypy.ini.jinja b/template/mypy.ini.jinja
@@ -1,2 +1,3 @@
 [mypy]
 disable_error_code = import-untyped
+exclude = (^|/)\.(snakemake|pixi)(/|$)
diff --git a/template/pixi.toml.jinja b/template/pixi.toml.jinja
@@ -13,6 +13,7 @@ clio-tools = ">=2025.03.03"
 conda = ">=25.0.0"
 ipdb = ">=0.13.13"
 ipykernel = ">=6.29.5"
+jsonschema = ">=4.0.0"
 mypy = ">=1.15.0"
 pytest = ">=8.3.5"
 python = ">=3.12"
@@ -21,17 +22,4 @@ snakefmt = ">=0.10.2"
 snakemake-minimal = ">=8.29.0"
 
 [tasks]
-test = {cmd = "pytest tests/"}
-
-[feature.docs.dependencies]
-mkdocs-material = ">=9.6.7"
-
-[feature.docs.pypi-dependencies]
-mkdocs-mermaid2-plugin = ">=1.2.1"
-
-[feature.docs.tasks]
-serve-docs = {cmd = "mkdocs serve"}
-build-docs = {cmd = "mkdocs build"}
-
-[environments]
-docs = ["docs"]
+test-integration = {cmd = "pytest tests/clio_test.py"}
diff --git a/template/tests/clio_test.py.jinja b/template/tests/clio_test.py.jinja
@@ -40,15 +40,15 @@ def test_standard_file_existance(module_path, file):
 def test_snakemake_all_failure(module_path):
     """The snakemake 'all' rule should return an error by default."""
     process = subprocess.run(
-        "snakemake --cores 4", shell=True, cwd=module_path, capture_output=True
+        "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True
     )
     assert "INVALID (missing locally)" in str(process.stderr)
 
 
 def test_snakemake_integration_testing(module_path):
     """Run a light-weight test simulating someone using this module."""
     assert subprocess.run(
-        "snakemake --use-conda --cores 4",
+        "snakemake --use-conda --cores 1",
         shell=True,
         check=True,
         cwd=module_path / "tests/integration",
diff --git a/template/tests/integration/{% if true %}Snakefile{% endif %}.jinja b/template/tests/integration/{% if true %}Snakefile{% endif %}.jinja
@@ -5,7 +5,7 @@ configfile: workflow.source_path("./test_config.yaml")
 rule create_external_input:
     message: "Example of a rule external to your module."
     output:
-        text_file = "results/{{module_short_name}}/resources/user/user_message.md"
+        text_file = "results/integration_test/resources/user/user_message.md"
     run:
         from pathlib import Path
         from textwrap import dedent
@@ -23,7 +23,7 @@ rule create_external_input:
 module {{module_short_name}}:
     snakefile: "../../workflow/Snakefile"
     config: config["{{module_short_name}}"]
-    prefix: "results/{{module_short_name}}/"
+    prefix: "results/integration_test/"
 
 # rename all module rules with a prefix, to avoid naming conflicts.
 use rule * from {{module_short_name}} as {{module_short_name}}_*
@@ -33,4 +33,4 @@ rule all:
     message: "A generic test case for your module."
     default_target: True
     input:
-        "results/{{module_short_name}}/results/combined_text.md"
+        "results/integration_test/results/combined_text.md"
diff --git a/template/workflow/Snakefile b/template/workflow/Snakefile
@@ -21,6 +21,10 @@ with open(workflow.source_path("internal/settings.yaml"), "r") as f:
 include: "rules/automatic.smk"
 include: "rules/dummy.smk"
 
+# Add additional files to be delivered alongside the workflow here.
+# This is needed e.g. for python code files that are never
+# explicitly used as a script or input in a snakemake rule.
+# e.g.: workflow.source_path("scripts/_utils.py")
 
 rule all:
     message:
diff --git a/tests/template_test.py b/tests/template_test.py
@@ -35,14 +35,11 @@ def pixi_built(template_project):
     return template_project
 
 
-def test_mkdocs_build(pixi_built):
-    """The template's mkdocs should build without issues."""
-    assert subprocess.run("pixi run build-docs", shell=True, check=True, cwd=pixi_built)
-
-
 def test_pytest(pixi_built):
     """The template's tests should pass by default."""
-    assert subprocess.run("pixi run test", shell=True, check=True, cwd=pixi_built)
+    assert subprocess.run(
+        "pixi run test-integration", shell=True, check=True, cwd=pixi_built
+    )
 
 
 def test_linting(pixi_built):

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`[mypy]`
`2`	`2`	`disable_error_code = import-untyped`
	`3`	`+exclude = (^\|/)\.(snakemake\|pixi)(/\|$)`