diff --git a/.github/workflows/reproduction.yaml b/.github/workflows/reproduction.yaml index 5f35954..4b87cb2 100644 --- a/.github/workflows/reproduction.yaml +++ b/.github/workflows/reproduction.yaml @@ -1,5 +1,11 @@ name: Reproduction -on: [push, pull_request] +on: + schedule: + - cron: "0 3 8 * *" # Runs every eighth day of the month at 3am. + push: + branches: + - main + pull_request: defaults: run: shell: bash -l {0} @@ -8,61 +14,67 @@ jobs: name: Reproduce the default demo analysis runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup cookiecutter environment - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.10" + python-version: 3.12 add-pip-as-python-dependency: true - name: Install cookiecutter run: pip install cookiecutter - name: Apply cookiecutter run: cookiecutter . --no-input --directory default - name: Setup Snakemake environment - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.10" - mamba-version: "*" + python-version: 3.12 activate-environment: reproducible-research-project environment-file: reproducible-research-project/environment.yaml - name: Reproduce results run: | cd reproducible-research-project - snakemake --cores 1 --use-conda + snakemake - name: Generate DAG run: | cd reproducible-research-project - snakemake --cores 1 --use-conda -f dag + snakemake dag + - name: Archive results + run: | + cd reproducible-research-project + snakemake archive run_cluster_workflow: name: Reproduce the cluster demo analysis (run locally only) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup cookiecutter environment - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.10" + python-version: 3.12 add-pip-as-python-dependency: true - name: Install cookiecutter run: pip install cookiecutter - name: Apply cookiecutter run: cookiecutter . --no-input --directory cluster - name: Setup Snakemake environment - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.10" - mamba-version: "*" + python-version: 3.12 activate-environment: reproducible-research-project environment-file: reproducible-research-project/environment.yaml - name: Reproduce results run: | cd reproducible-research-project - snakemake --cores 1 --use-conda + snakemake - name: Generate DAG run: | cd reproducible-research-project - snakemake --cores 1 --use-conda -f dag + snakemake dag + - name: Archive results + run: | + cd reproducible-research-project + snakemake archive diff --git a/LICENSE.md b/LICENSE.md index baa2b02..8de62eb 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2017-2021 Tim Tröndle +Copyright (c) 2017-2024 Tim Tröndle Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/README.md b/README.md index 414ea12..9be1645 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,11 @@ This repository provides [cookiecutter](http://cookiecutter.readthedocs.io) templates for reproducible research projects. The templates do not attempt to be generic, but have a clear and opinionated focus. -Projects build with these templates aim at full automation, and use `Python 3.10`, `mamba/conda`, `Git`, `Snakemake`, and `pandoc` to create a HTML report out of raw data, code, and `Markdown` text. Fork, clone, or download this repository on GitHub if you want to change any of these. +Projects build with these templates aim at full automation, and use `Python 3.12`, `conda`, `Git`, `Snakemake`, and `pandoc` to create a HTML and PDF report out of raw data, code, and `Markdown` text. Fork, clone, or download this repository on GitHub if you want to change any of these. -The template includes a few lines of code as a demo to allow you to create a HTML report out of made-up simulation results right away. Read the `README.md` in the generated repository to see how. +The template includes a few lines of code as a demo to allow you to create a report out of made-up simulation results right away. Read the `README.md` in the generated repository to see how. + +These templates are developed on macOS and tested on Linux. They may work with Windows Subsystem for Linux, but Windows is not actively supported. ## Template types @@ -37,6 +39,7 @@ Parameter | Description `author` | Your name. `institute` | The name of your institute, used for report metadata. `short_description` | A short description of the project, used for documentation and report. +`path_to_conda_envs` | The path to the directory hosting your conda envs (leave untouched for Snakemake default). The `cluster` template requires the following parameter values in addition: @@ -44,7 +47,8 @@ Parameter | Description --- | --- `cluster_url` | The address of the cluster to allow syncing to and from the cluster. `cluster_base_dir` | The base path for the project on the cluster (default: `~/`). -`cluster_type` | The type of job scheduler used on the cluster. Currently, only LSF is supported. +`cluster_type` | The type of job scheduler used on the cluster. Currently, only Slurm is supported. +`slurm_account` | The user account on Slurm. ## Project Structure @@ -58,6 +62,9 @@ The generated repository will have the following structure: │ ├── default.yaml <- Default execution environment. │ ├── report.yaml <- Environment for compilation of the report. │ └── test.yaml <- Environment for executing tests. +├── profiles <- Snakemake profiles. +│ └── default <- Default Snakemake profile folder. +│ └── config.yaml <- Default Snakemake profile. ├── report <- All files creating the final report, usually text and figures. │ ├── apa.csl <- Citation style definition to be used in the report. │ ├── literature.yaml <- Bibliography file for the report. @@ -70,7 +77,7 @@ The generated repository will have the following structure: ├── tests <- Automatic tests of the source code go in here. │ └── test_model.py <- Demo file. ├── .editorconfig <- Editor agnostic configuration settings. -├── .flake8 <- Linting settings for flake8. +├── .ruff <- Linter and formatter settings for ruff. ├── .gitignore ├── environment.yaml <- A file to create an environment to execute your project in. ├── LICENSE.md <- MIT license description @@ -81,12 +88,11 @@ The generated repository will have the following structure: `cluster` templates additionally contain the following files: ``` -├── config -│ └── cluster <- Cluster configuration. -│ ├── cluster-config.yaml <- A Snakemake cluster-config file. -│ └── config.yaml <- A set of Snakemake command-line parameters for cluster execution. ├── envs │ └── shell.yaml <- An environment for shell rules. +├── profiles +│ └── cluster <- Cluster Snakemake profile folder. +│ └── config.yaml <- Cluster Snakemake profile. ├── rules │ └── sync.yaml <- Snakemake rules to sync to and from the cluster. ├── .syncignore-receive <- Build files to ignore when receiving from the cluster. diff --git a/cluster/cookiecutter.json b/cluster/cookiecutter.json index 42a5c72..7bbc2e6 100644 --- a/cluster/cookiecutter.json +++ b/cluster/cookiecutter.json @@ -4,8 +4,11 @@ "author": "Your name", "institute": "Your institution", "short_description": "A short description of this project.", + "path_to_conda_envs": "Snakemake-default", "cluster_url": "cluster.example.org", "cluster_base_dir": "~/{{ cookiecutter.project_short_name }}", - "cluster_type": ["LSF"], - "_add_cluster_infrastructure": true + "cluster_type": ["Slurm"], + "slurm_account": "Your slurm account", + "_add_cluster_infrastructure": true, + "_jinja2_env_vars": {"lstrip_blocks": true, "trim_blocks": true} } diff --git a/cluster/{{cookiecutter.project_short_name}}/.flake8 b/cluster/{{cookiecutter.project_short_name}}/.flake8 deleted file mode 120000 index 2d4ea7c..0000000 --- a/cluster/{{cookiecutter.project_short_name}}/.flake8 +++ /dev/null @@ -1 +0,0 @@ -../../default/{{cookiecutter.project_short_name}}/.flake8 \ No newline at end of file diff --git a/cluster/{{cookiecutter.project_short_name}}/.ruff.toml b/cluster/{{cookiecutter.project_short_name}}/.ruff.toml new file mode 120000 index 0000000..f0f9d98 --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/.ruff.toml @@ -0,0 +1 @@ +../../default/{{cookiecutter.project_short_name}}/.ruff.toml \ No newline at end of file diff --git a/cluster/{{cookiecutter.project_short_name}}/.syncignore-send b/cluster/{{cookiecutter.project_short_name}}/.syncignore-send index 8e7f489..cd4f440 100644 --- a/cluster/{{cookiecutter.project_short_name}}/.syncignore-send +++ b/cluster/{{cookiecutter.project_short_name}}/.syncignore-send @@ -7,4 +7,5 @@ __pycache__ .vscode .DS_Store build +archive notebooks diff --git a/cluster/{{cookiecutter.project_short_name}}/config/cluster/cluster-config.yaml b/cluster/{{cookiecutter.project_short_name}}/config/cluster/cluster-config.yaml deleted file mode 100644 index 7403214..0000000 --- a/cluster/{{cookiecutter.project_short_name}}/config/cluster/cluster-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -__default__: - runtime: 10 - cores: 1 - memory: 16000 - output: "build/logs/{rule}.{wildcards}.log" - name: "{rule}.{wildcards}" diff --git a/cluster/{{cookiecutter.project_short_name}}/config/cluster/config.yaml b/cluster/{{cookiecutter.project_short_name}}/config/cluster/config.yaml deleted file mode 100644 index 485ecc7..0000000 --- a/cluster/{{cookiecutter.project_short_name}}/config/cluster/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -cluster: bsub -oo {cluster.output} -W {cluster.runtime} -n {cluster.cores} -R "rusage[mem={cluster.memory}]" -J {cluster.name} < -jobs: 999 -cluster-config: "config/cluster/cluster-config.yaml" -local-cores: 1 -latency-wait: 60 diff --git a/cluster/{{cookiecutter.project_short_name}}/profiles/cluster/config.yaml b/cluster/{{cookiecutter.project_short_name}}/profiles/cluster/config.yaml new file mode 100644 index 0000000..ee5ce35 --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/profiles/cluster/config.yaml @@ -0,0 +1,14 @@ +executor: slurm +jobs: 999 +local-cores: 1 +cores: 10 +latency-wait: 60 +software-deployment-method: conda +{% if cookiecutter.path_to_conda_envs != "Snakemake-default" %} +conda-prefix: {{cookiecutter.path_to_conda_envs}} +{% endif %} +default-resources: + - runtime=10 + - mem_mb_per_cpu=16000 + - disk_mb=1000 + - slurm_account={{cookiecutter.slurm_account}} diff --git a/cluster/{{cookiecutter.project_short_name}}/profiles/default/config.yaml b/cluster/{{cookiecutter.project_short_name}}/profiles/default/config.yaml new file mode 120000 index 0000000..7d14879 --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/profiles/default/config.yaml @@ -0,0 +1 @@ +../../../../default/{{cookiecutter.project_short_name}}/profiles/default/config.yaml \ No newline at end of file diff --git a/cluster/{{cookiecutter.project_short_name}}/rules/sync.smk b/cluster/{{cookiecutter.project_short_name}}/rules/sync.smk index e24d14b..2e2b2fc 100644 --- a/cluster/{{cookiecutter.project_short_name}}/rules/sync.smk +++ b/cluster/{{cookiecutter.project_short_name}}/rules/sync.smk @@ -14,7 +14,6 @@ rule send: {params.url}:{params.cluster_base_dir} """ - rule receive: message: "Receive build changes from cluster" params: @@ -27,7 +26,7 @@ rule receive: conda: "../envs/shell.yaml" shell: """ - rsync -avzh --progress --delete -r --exclude-from={params.receive_ignore} \ + rsync -avzhL --progress --delete -r --exclude-from={params.receive_ignore} \ {params.url}:{params.cluster_build_dir} {params.local_results_dir} """ diff --git a/cluster/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi b/cluster/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi new file mode 120000 index 0000000..9312e6e --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi @@ -0,0 +1 @@ +../../../default/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi \ No newline at end of file diff --git a/cluster/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua b/cluster/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua new file mode 120000 index 0000000..837e094 --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua @@ -0,0 +1 @@ +../../../../default/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua \ No newline at end of file diff --git a/cluster/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi b/cluster/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi new file mode 120000 index 0000000..90b9045 --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi @@ -0,0 +1 @@ +../../../default/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi \ No newline at end of file diff --git a/cluster/{{cookiecutter.project_short_name}}/tests/test_runner.py b/cluster/{{cookiecutter.project_short_name}}/tests/test_runner.py new file mode 120000 index 0000000..9db881d --- /dev/null +++ b/cluster/{{cookiecutter.project_short_name}}/tests/test_runner.py @@ -0,0 +1 @@ +../../../default/{{cookiecutter.project_short_name}}/tests/test_runner.py \ No newline at end of file diff --git a/default/cookiecutter.json b/default/cookiecutter.json index 3694549..f4b121f 100644 --- a/default/cookiecutter.json +++ b/default/cookiecutter.json @@ -4,5 +4,7 @@ "author": "Your name", "institute": "Your institution", "short_description": "A short description of this project.", - "_add_cluster_infrastructure": false + "path_to_conda_envs": "Snakemake-default", + "_add_cluster_infrastructure": false, + "_jinja2_env_vars": {"lstrip_blocks": true, "trim_blocks": true} } diff --git a/default/{{cookiecutter.project_short_name}}/.flake8 b/default/{{cookiecutter.project_short_name}}/.flake8 deleted file mode 100644 index 070188f..0000000 --- a/default/{{cookiecutter.project_short_name}}/.flake8 +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -max-line-length = 119 -ignore = E261 -exclude = Snakefile, *.smk -builtins = snakemake # for using the snakemake injection in scripts diff --git a/default/{{cookiecutter.project_short_name}}/.gitignore b/default/{{cookiecutter.project_short_name}}/.gitignore index cc96b01..47d8e46 100644 --- a/default/{{cookiecutter.project_short_name}}/.gitignore +++ b/default/{{cookiecutter.project_short_name}}/.gitignore @@ -1,5 +1,6 @@ # generated files build/ +archive/ ## Core latex/pdflatex auxiliary files: *.aux diff --git a/default/{{cookiecutter.project_short_name}}/.ruff.toml b/default/{{cookiecutter.project_short_name}}/.ruff.toml new file mode 100644 index 0000000..e47ad48 --- /dev/null +++ b/default/{{cookiecutter.project_short_name}}/.ruff.toml @@ -0,0 +1,36 @@ +line-length = 88 +preview = true # required to activate many pycodestyle errors and warnings as of 2024-05-01 +builtins = ["snakemake"] + +[format] +quote-style = "double" +indent-style = "space" +docstring-code-format = false +line-ending = "auto" + +[lint] +select = [ + # pycodestyle errors + "E", + # pycodestyle warnings + "W", + # Pyflakes + "F", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # isort + "I", +] +ignore = [ + # here and below, rules are redundant with formatter, see + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "E501", + "W191", + "E111", + "E114", + "E117", +] diff --git a/default/{{cookiecutter.project_short_name}}/README.md b/default/{{cookiecutter.project_short_name}}/README.md index f3ee1c0..bbaf943 100644 --- a/default/{{cookiecutter.project_short_name}}/README.md +++ b/default/{{cookiecutter.project_short_name}}/README.md @@ -6,13 +6,13 @@ This repository contains the entire scientific project, including code and repor ## Getting ready -You need [mamba](https://mamba.readthedocs.io/en/latest/) to run the analysis. Using mamba, you can create an environment from within you can run it: +You need [conda](https://conda.org) to run the analysis. Using conda, you can create an environment from within you can run it: - mamba env create -f environment.yaml + conda env create -f environment.yaml --no-default-packages ## Run the analysis - snakemake --cores 1 --use-conda + snakemake This will run all analysis steps to reproduce results and eventually build the report. @@ -20,14 +20,14 @@ You can also run certain parts only by using other `snakemake` rules; to get a l To generate a PDF of the dependency graph of all steps `build/dag.pdf` run: - snakemake --use-conda --cores 1 -f dag + snakemake dag -{% if cookiecutter._add_cluster_infrastructure == True -%} +{% if cookiecutter._add_cluster_infrastructure == True %} ## Run on a cluster -You may want to run the workflow on a cluster. While you can run on [any cluster that is supported by Snakemake](https://snakemake.readthedocs.io/en/stable/executing/cluster.html), the workflow currently supports [LSF](https://en.wikipedia.org/wiki/Platform_LSF) clusters only. To run the workflow on a LSF cluster, use the following command: +You may want to run the workflow on a cluster. While you can run on [any cluster that is supported by Snakemake](https://snakemake.readthedocs.io/en/stable/executing/cluster.html), the workflow currently supports [Slurm](https://en.wikipedia.org/wiki/Slurm_Workload_Manager) clusters only. To run the workflow on a Slurm cluster, use the following command: - snakemake --use-conda --profile config/cluster + snakemake --profile profiles/cluster If you want to run on another cluster, read [snakemake's documentation on cluster execution](https://snakemake.readthedocs.io/en/stable/executable.html#cluster-execution) and take `config/cluster` as a starting point. @@ -35,22 +35,24 @@ If you want to run on another cluster, read [snakemake's documentation on cluste You may want to work locally (to change configuration parameters, add modules etc), but execute remotely on the cluster. This workflow supports you in working this way through three Snakemake rules: `send`, `receive`, and `clean_cluster_results`. It works like the following. -First, start local and make sure the `cluster-sync` configuration parameters fit your environment. Next, run `snakemake --use-conda send` to send the entire repository to your cluster. On the cluster, execute the workflow with Snakemake (see above). After the workflow has finished, download results by locally running `snakemake --use-conda receive`. By default, this will download results into `build/cluster`. +First, start local and make sure the `cluster-sync` configuration parameters fit your environment. Next, run `snakemake send` to send the entire repository to your cluster. On the cluster, execute the workflow with Snakemake (see above). After the workflow has finished, download results by locally running `snakemake receive`. By default, this will download results into `build/cluster`. -This workflow works iteratively too. After analysing your cluster results locally, you may want to make changes locally, send these changes to the cluster (`snakemake --use-conda send`), rerun on the cluster, and download updated results (`snakemake --use-conda receive`). +This workflow works iteratively too. After analysing your cluster results locally, you may want to make changes locally, send these changes to the cluster (`snakemake send`), rerun on the cluster, and download updated results (`snakemake receive`). -To remove cluster results on your local machine, run `snakemake --use-conda clean_cluster_results`. -{%- endif %} +To remove cluster results on your local machine, run `snakemake clean_cluster_results`. +{% endif %} ## Be notified of build successes or fails - As the execution of this workflow may take a while, you can be notified whenever the execution terminates either successfully or unsuccessfully. Notifications are sent by email. To activate notifications, add the email address of the recipient to the configuration key `email`. You can add the key to your configuration file, or you can run the workflow the following way to receive notifications: +As the execution of this workflow may take a while, you can be notified whenever the execution terminates either successfully or unsuccessfully. Notifications are sent by the webservice [Pushcut](https://pushcut.io/) for which you need a free account. To activate notifications, add your Pushcut secret to the configuration using the configuration key `pushcut_secret`. You can add the key to your configuration file, or you can run the workflow the following way to receive notifications: - snakemake --cores 1 --use-conda --config email= + snakemake --config pushcut_secret= + +This workflow will then trigger the Pushcut notifications `snakemake_succeeded` and `snakemake_failed`. ## Run the tests - snakemake --use-conda --cores 1 test + snakemake test ## Repo structure @@ -60,6 +62,7 @@ To remove cluster results on your local machine, run `snakemake --use-conda clea * `envs`: contains execution environments * `tests`: contains the test code * `config`: configurations used in the study +* `profiles`: Snakemake execution profiles * `data`: place for raw data * `build`: will contain all results (does not exist initially) diff --git a/default/{{cookiecutter.project_short_name}}/Snakefile b/default/{{cookiecutter.project_short_name}}/Snakefile index 01cbc3b..7923c11 100644 --- a/default/{{cookiecutter.project_short_name}}/Snakefile +++ b/default/{{cookiecutter.project_short_name}}/Snakefile @@ -1,32 +1,45 @@ -PANDOC = "pandoc --filter pantable --filter pandoc-fignos --filter pandoc-tablenos --citeproc" +from snakemake.utils import min_version +from pathlib import Path +import requests +PANDOC = "pandoc --filter pantable --filter pandoc-crossref --citeproc -f markdown+mark" configfile: "config/default.yaml" -{% if cookiecutter._add_cluster_infrastructure == True -%} +{% if cookiecutter._add_cluster_infrastructure == True %} include: "./rules/sync.smk" -localrules: all, report, clean -{%- endif %} -{% if cookiecutter._add_cluster_infrastructure == True -%} +SNAKEMAKE_LOG_DIR = Path(".snakemake/log") +SLURM_LOG_DIR = Path(".snakemake/slurm_logs") +SNAKEMAKE_LOG_PUBLISH_DIR = Path("build/logs/snakemake") +SLURM_LOG_PUBLISH_DIR = Path("build/logs/slurm") + +{% endif %} +min_version("9.6") + +{% if cookiecutter._add_cluster_infrastructure == True %} onstart: - shell("mkdir -p build/logs") -{%- endif %} + prepare_log_directories() +{% endif %} onsuccess: - if "email" in config.keys(): - shell("echo "" | mail -s '{{cookiecutter.project_short_name}} succeeded' {config[email]}") + if "pushcut_secret" in config.keys(): + trigger_pushcut(event_name="snakemake_succeeded", secret=config["pushcut_secret"]) onerror: - if "email" in config.keys(): - shell("echo "" | mail -s '{{cookiecutter.project_short_name}} failed' {config[email]}") + if "pushcut_secret" in config.keys(): + trigger_pushcut(event_name="snakemake_failed", secret=config["pushcut_secret"]) + rule all: message: "Run entire analysis and compile report." + {% if cookiecutter._add_cluster_infrastructure == True %} + localrule: True + {% endif %} input: "build/report.html", - "build/test-report.html" + "build/report.pdf", + "build/test.success" rule run: message: "Runs the demo model." - input: "scripts/model.py" params: slope = config["slope"], x0 = config["x0"] @@ -38,7 +51,6 @@ rule run: rule plot: message: "Visualises the demo results." input: - scripts = "scripts/vis.py", results = rules.run.output output: "build/plot.png" conda: "envs/default.yaml" @@ -48,9 +60,9 @@ rule plot: def pandoc_options(wildcards): suffix = wildcards["suffix"] if suffix == "html": - return "--self-contained --to html5" + return "--embed-resources --standalone --to html5 --mathml" elif suffix == "pdf": - return "--pdf-engine weasyprint" + return "--pdf-engine weasyprint --lua-filter='../scripts/math/math-katex.lua'" elif suffix == "docx": return [] else: @@ -66,6 +78,7 @@ rule report: "report/apa.csl", "report/reset.css", "report/report.css", + "scripts/math/math-katex.lua", rules.plot.output params: options = pandoc_options output: "build/report.{suffix}" @@ -82,21 +95,32 @@ rule report: """ +rule dag_dot: + {% if cookiecutter._add_cluster_infrastructure == True %} + localrule: True + {% endif %} + output: temp("build/dag.dot") + shell: + "snakemake --rulegraph > {output}" + + rule dag: - message: "Plot dependency graph of the workflow." - output: - dot = "build/dag.dot", - pdf = "build/dag.pdf" - conda: "envs/dag.yaml" - shell: - """ - snakemake --rulegraph > {output.dot} - dot -Tpdf -o {output.pdf} {output.dot} - """ + message: "Plot dependency graph of the workflow." + {% if cookiecutter._add_cluster_infrastructure == True %} + localrule: True + {% endif %} + input: rules.dag_dot.output[0] + # Output is deliberately omitted so rule is executed each time. + conda: "envs/dag.yaml" + shell: + "dot -Tpdf {input} -o build/dag.pdf" rule clean: # removes all generated results message: "Remove all build results but keep downloaded data." + {% if cookiecutter._add_cluster_infrastructure == True %} + localrule: True + {% endif %} run: import shutil @@ -104,8 +128,62 @@ rule clean: # removes all generated results print("Data downloaded to data/ has not been cleaned.") +rule archive: + message: "Package, zip, and move entire build." + params: + push_from_directory = config["push"]["from"], + push_to_directory = config["push"]["to"], + exclude_paths = config["push"]["exclude-paths"] + run: + from datetime import datetime + from pathlib import Path + import tarfile + + today = datetime.today().strftime('%Y-%m-%d') + from_folder = Path(params.push_from_directory) + to_folder = Path(params.push_to_directory).expanduser() + build_archive_filename = to_folder / f"{{ cookiecutter.project_short_name }}-{today}.gz" + + to_folder.mkdir(parents=True, exist_ok=True) + assert to_folder.is_dir(), f"Archive folder {to_folder} does not exist." + + exclude_paths = params.exclude_paths if params.exclude_paths else [] + + with tarfile.open(build_archive_filename, "w:gz") as tar: + tar.add(from_folder, filter=lambda x: None if x.name in exclude_paths else x) + + rule test: - conda: "envs/test.yaml" - output: "build/test-report.html" - shell: - "py.test --html={output} --self-contained-html" + # To add more tests, do + # (1) Add to-be-tested workflow outputs as inputs to this rule. + # (2) Turn them into pytest fixtures in tests/test_runner.py. + # (3) Create or reuse a test file in tests/my-test.py and use fixtures in tests. + message: "Run tests" + input: + test_dir = "tests", + tests = map(str, Path("tests").glob("**/test_*.py")), + model_results = rules.run.output[0], + log: "build/test-report.html" + output: "build/test.success" + conda: "./envs/test.yaml" + script: "./tests/test_runner.py" + + +def trigger_pushcut(event_name: str, secret: str) -> None: + """Trigger a Pushcut notification.""" + response = requests.post( + f'https://api.pushcut.io/{secret}/notifications/{event_name}' + ) + response.raise_for_status() + + +{% if cookiecutter._add_cluster_infrastructure == True %} +def prepare_log_directories() -> None: + """Create symlinks to Snakemake log directories.""" + if SLURM_LOG_DIR.resolve().exists() and not SLURM_LOG_PUBLISH_DIR.resolve().exists(): + SLURM_LOG_PUBLISH_DIR.parent.mkdir(parents=True, exist_ok=True) + SLURM_LOG_PUBLISH_DIR.symlink_to(SLURM_LOG_DIR.resolve()) + if SNAKEMAKE_LOG_DIR.resolve().exists() and not SNAKEMAKE_LOG_PUBLISH_DIR.resolve().exists(): + SNAKEMAKE_LOG_PUBLISH_DIR.parent.mkdir(parents=True, exist_ok=True) + SNAKEMAKE_LOG_PUBLISH_DIR.symlink_to(SNAKEMAKE_LOG_DIR.resolve()) +{% endif %} diff --git a/default/{{cookiecutter.project_short_name}}/config/default.yaml b/default/{{cookiecutter.project_short_name}}/config/default.yaml index dcb63fe..4332bea 100644 --- a/default/{{cookiecutter.project_short_name}}/config/default.yaml +++ b/default/{{cookiecutter.project_short_name}}/config/default.yaml @@ -1,11 +1,15 @@ # The right place for all your configuration values. slope: 4 x0: 5 -{% if cookiecutter._add_cluster_infrastructure == True -%} +push: + from: build + to: archive + exclude-paths: +{% if cookiecutter._add_cluster_infrastructure == True %} cluster-sync: url: {{ cookiecutter.cluster_url }} send-ignore: .syncignore-send receive-ignore: .syncignore-receive cluster-base-dir: {{ cookiecutter.cluster_base_dir }} local-results-dir: build/cluster -{%- endif %} +{% endif %} diff --git a/default/{{cookiecutter.project_short_name}}/environment.yaml b/default/{{cookiecutter.project_short_name}}/environment.yaml index d050bef..2aaa3bd 100644 --- a/default/{{cookiecutter.project_short_name}}/environment.yaml +++ b/default/{{cookiecutter.project_short_name}}/environment.yaml @@ -3,6 +3,8 @@ channels: - conda-forge - bioconda dependencies: - - python=3.10 - - flake8=3.8.3 - - snakemake-minimal=6.5.3 + - python=3.12 + - snakemake-minimal=9.6.2 + {% if cookiecutter._add_cluster_infrastructure == True %} + - snakemake-executor-plugin-slurm=1.4.0 + {% endif %} diff --git a/default/{{cookiecutter.project_short_name}}/envs/dag.yaml b/default/{{cookiecutter.project_short_name}}/envs/dag.yaml index 1b998a0..cb46e7d 100644 --- a/default/{{cookiecutter.project_short_name}}/envs/dag.yaml +++ b/default/{{cookiecutter.project_short_name}}/envs/dag.yaml @@ -1,8 +1,5 @@ name: dag channels: - conda-forge - - bioconda dependencies: - - python=3.10 - - snakemake-minimal=6.5.3 - graphviz=2.50.0 diff --git a/default/{{cookiecutter.project_short_name}}/envs/default.yaml b/default/{{cookiecutter.project_short_name}}/envs/default.yaml index 1d86c48..da05daf 100644 --- a/default/{{cookiecutter.project_short_name}}/envs/default.yaml +++ b/default/{{cookiecutter.project_short_name}}/envs/default.yaml @@ -2,8 +2,9 @@ name: default channels: - conda-forge dependencies: - - python=3.10 - - numpy=1.22.3 - - pandas=1.4.1 - - matplotlib=3.5.1 - - seaborn=0.11.2 + - python=3.12 + - ipdb=0.13.13 # replaces default debugger with IPython debugger + - numpy=2.3.0 + - pandas=2.3.0 + - matplotlib=3.10.3 + - seaborn=0.13.2 diff --git a/default/{{cookiecutter.project_short_name}}/envs/report.yaml b/default/{{cookiecutter.project_short_name}}/envs/report.yaml index f4bd6c8..eaf209d 100644 --- a/default/{{cookiecutter.project_short_name}}/envs/report.yaml +++ b/default/{{cookiecutter.project_short_name}}/envs/report.yaml @@ -2,15 +2,15 @@ name: report channels: - conda-forge dependencies: - - python=3.9 - - pango=1.50.3 - - cairocffi==1.1.0 - - psutil=5.7.2 - - weasyprint==54.0 - - pandoc=2.11.3.2 - - pip=22.0.4 + - python=3.12 + - pango=1.56.3 + - cairocffi=1.7.1 + - psutil=7.0.0 + - weasyprint=62.3 + - pandoc=3.6.2 + - pandoc-crossref=0.3.18.1 + - katex=0.16.9 + - brotlipy=0.7.0 + - pip=25.1.1 - pip: - - pandoc-xnos==2.5.0 - - pandoc-tablenos==2.3.0 - - pandoc-fignos==2.4.0 - - pantable==0.13.1 + - pantable==0.14.2 diff --git a/default/{{cookiecutter.project_short_name}}/envs/test.yaml b/default/{{cookiecutter.project_short_name}}/envs/test.yaml index 3c049ef..ed6b9db 100644 --- a/default/{{cookiecutter.project_short_name}}/envs/test.yaml +++ b/default/{{cookiecutter.project_short_name}}/envs/test.yaml @@ -2,11 +2,8 @@ name: test channels: - conda-forge dependencies: - - python=3.10 - - numpy=1.22.3 - - pandas=1.4.1 - - pytest=6.2.5 - - pytest-html=2.1.1 - - pip=22.0.4 - - pip: - - pytest-pythonpath # adds the root path "./" to PYTHONPATH + - python=3.12 + - numpy=2.3.0 + - pandas=2.3.0 + - pytest=8.4.1 + - pytest-html=4.1.1 diff --git a/default/{{cookiecutter.project_short_name}}/profiles/default/config.yaml b/default/{{cookiecutter.project_short_name}}/profiles/default/config.yaml new file mode 100644 index 0000000..4161a4b --- /dev/null +++ b/default/{{cookiecutter.project_short_name}}/profiles/default/config.yaml @@ -0,0 +1,5 @@ +software-deployment-method: conda +{% if cookiecutter.path_to_conda_envs != "Snakemake-default" %} +conda-prefix: {{cookiecutter.path_to_conda_envs}} +{% endif %} +cores: 1 diff --git a/default/{{cookiecutter.project_short_name}}/report/pandoc-metadata.yaml b/default/{{cookiecutter.project_short_name}}/report/pandoc-metadata.yaml index 2c95c85..a20f834 100644 --- a/default/{{cookiecutter.project_short_name}}/report/pandoc-metadata.yaml +++ b/default/{{cookiecutter.project_short_name}}/report/pandoc-metadata.yaml @@ -3,7 +3,7 @@ title: {{cookiecutter.project_name}} author: - {{cookiecutter.author}} institute: {{cookiecutter.institute}} -tags: +keywords: - reproducible-research abstract: | {{cookiecutter.short_description}} @@ -12,10 +12,23 @@ csl: apa.csl link-citations: True css: - reset.css + - https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/katex.min.css - report.css lang: en-GB -xnos-cleveref: True -fignos-plus-name: Figure -tablenos-plus-name: Table +figureTemplate: $$figureTitle$$ $$i$$$$titleDelim$$ $$t$$ +tableTemplate: $$tableTitle$$ $$i$$$$titleDelim$$ $$t$$ +figPrefix: + - "Figure" + - "Figures" +tblPrefix: + - "Table" + - "Tables" +eqnPrefix: + - "Equation" + - "Equations" +secPrefix: + - "Section" + - "Sections" +linkReferences: True date: {% now 'utc', '%Y-%m-%d' %} --- diff --git a/default/{{cookiecutter.project_short_name}}/report/report.md b/default/{{cookiecutter.project_short_name}}/report/report.md index aac1a6f..2bad1f1 100644 --- a/default/{{cookiecutter.project_short_name}}/report/report.md +++ b/default/{{cookiecutter.project_short_name}}/report/report.md @@ -2,12 +2,20 @@ ... +# Methods + +@eq:demo is a math equation. + +$$ +x = y +$$ {{ '{#eq:demo}' }} + # Results @fig:linear-model shows the results we found: ![The linear model.](../build/plot.png){ #fig:linear-model .class} -This results confirm former findings in refs.\ [@Trondle:2019] and [@Trondle:2020]. +These results confirm former findings in refs.\ [@Trondle:2019] and [@Trondle:2020]. ==Double check== # References diff --git a/default/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi b/default/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi new file mode 100644 index 0000000..f4fa58e --- /dev/null +++ b/default/{{cookiecutter.project_short_name}}/scripts/__builtins__.pyi @@ -0,0 +1,2 @@ +class snakemake: + pass diff --git a/default/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua b/default/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua new file mode 100644 index 0000000..7d51096 --- /dev/null +++ b/default/{{cookiecutter.project_short_name}}/scripts/math/math-katex.lua @@ -0,0 +1,111 @@ +-- VERSION 2022-01-04 +-- DESCRIPTION +-- +-- This Lua filter for Pandoc converts LaTeX math with katex for insertion into the +-- output document in a standalone manner. SVG output is in any of the available +-- MathJax fonts. This is useful for server-side rendering. No Internet connection +-- is required when generating or viewing formulas, resulting in both absolute +-- privacy and offline, standalone robustness. +-- +-- REQUIREMENTS, USAGE & PRIVACY +-- +-- See: https://github.com/pandoc/lua-filters/tree/master/math-katex +-- +-- LICENSE +-- +-- Copyright (c) 2020-2022 Benjamin Abel +-- +-- MIT License +-- +-- Permission is hereby granted, free of charge, to any person obtaining a +-- copy of this software and associated documentation files (the "Software"), +-- to deal in the Software without restriction, including without limitation +-- the rights to use, copy, modify, merge, publish, distribute, sublicense, +-- and/or sell copies of the Software, and to permit persons to whom the +-- Software is furnished to do so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in +-- all copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +-- THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +-- DEALINGS IN THE SOFTWARE. +-- +-- The full path to the katex binary. +local katex_bin = 'katex' +local no_throw_on_error = false +local format = 'htmlAndMathml' + +function Meta(meta) + + katex_bin = tostring(meta.math_katex_bin or katex_bin) + no_throw_on_error = tostring(meta.math_katex_no_throw_on_error or no_throw_on_error) + format = tostring(meta.math_katex_format or format) + +end + +function Math(elem) + + -- TODO handle macros with header-includes + + local argumentlist = {'--format', format, '--no-throw-on-error', no_throw_on_error} + + -- The available options for katex are: + -- -V, --version output the version number + -- -d, --display-mode Render math in display mode, which puts the math in display style (so \int and \sum are large, for + -- example), and centers the math on the page on its own line. + -- -F, --format Determines the markup language of the output. + -- --leqno Render display math in leqno style (left-justified tags). + -- --fleqn Render display math flush left. + -- -t, --no-throw-on-error Render errors (in the color given by --error-color) instead of throwing a ParseError exception when + -- encountering an error. + -- -c, --error-color A color string given in the format 'rgb' or 'rrggbb' (no #). This option determines the color of errors + -- rendered by the -t option. + -- -m, --macro Define custom macro of the form '\foo:expansion' (use multiple -m arguments for multiple macros). + -- (default: []) + -- --min-rule-thickness Specifies a minimum thickness, in ems, for fraction lines, `\sqrt` top lines, `{array}` vertical lines, + -- `\hline`, `\hdashline`, `\underline`, `\overline`, and the borders of `\fbox`, `\boxed`, and + -- `\fcolorbox`. + -- -b, --color-is-text-color Makes \color behave like LaTeX's 2-argument \textcolor, instead of LaTeX's one-argument \color mode + -- change. + -- -S, --strict Turn on strict / LaTeX faithfulness mode, which throws an error if the input uses features that are not + -- supported by LaTeX. + -- -T, --trust Trust the input, enabling all HTML features such as \url. + -- -s, --max-size If non-zero, all user-specified sizes, e.g. in \rule{500em}{500em}, will be capped to maxSize ems. + -- Otherwise, elements and spaces can be arbitrarily large + -- -e, --max-expand Limit the number of macro expansions to the specified number, to prevent e.g. infinite macro loops. If + -- set to Infinity, the macro expander will try to fully expand as in LaTeX. + -- -f, --macro-file Read macro definitions, one per line, from the given file. + -- -i, --input Read LaTeX input from the given file. + -- -o, --output Write html output to the given file. + -- -h, --help display help for command + -- + + if (elem.mathtype == 'DisplayMath') then + -- Add the --display-mode argument to the argument list. + table.insert(argumentlist, 1, '--display-mode') + end + + -- Generate markup. + + local markup = pandoc.pipe(katex_bin, argumentlist, elem.text) + + -- remove \n at the end https://github.com/KaTeX/KaTeX/blob/16b4bd9c0c315220d41a610c903e1701ca9c1042/cli.js#L99 + markup = markup:sub(1, -2) + + return pandoc.RawInline('html', markup) + +end -- function + +-- Redefining the execution order only in html +if FORMAT == "html" then + return { { + Meta = Meta + }, { + Math = Math + } } +end diff --git a/default/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi b/default/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi new file mode 100644 index 0000000..f4fa58e --- /dev/null +++ b/default/{{cookiecutter.project_short_name}}/tests/__builtins__.pyi @@ -0,0 +1,2 @@ +class snakemake: + pass diff --git a/default/{{cookiecutter.project_short_name}}/tests/test_model.py b/default/{{cookiecutter.project_short_name}}/tests/test_model.py index e04ef96..c7579ca 100644 --- a/default/{{cookiecutter.project_short_name}}/tests/test_model.py +++ b/default/{{cookiecutter.project_short_name}}/tests/test_model.py @@ -1,6 +1,12 @@ -"""Test case for the model.""" -import scripts.model +"""Test case for the model results. +Pytest fixtures are provided from within the test runner. +""" -def test_model(): - assert scripts.model.linear_model(slope=1, x0=4, x=0) == 4 + +def test_model_results_at_0(model_results): + assert model_results[0] == 5 + + +def test_model_results_at_1(model_results): + assert model_results[1] == 9 diff --git a/default/{{cookiecutter.project_short_name}}/tests/test_runner.py b/default/{{cookiecutter.project_short_name}}/tests/test_runner.py new file mode 100644 index 0000000..070ea8e --- /dev/null +++ b/default/{{cookiecutter.project_short_name}}/tests/test_runner.py @@ -0,0 +1,38 @@ +import sys +from pathlib import Path + +import pandas as pd +import pytest + + +def run_test(snakemake): + exit_code = pytest.main( + [ + snakemake.input.test_dir, + f"--html={snakemake.log[0]}", + "--self-contained-html", + "--verbose" + ], + plugins=[ + _create_config_plugin(snakemake=snakemake) + ] + ) + if exit_code == 0: + Path(snakemake.output[0]).touch() + sys.exit(exit_code) + + +def _create_config_plugin(snakemake): + """Creates fixtures from Snakemake configuration.""" + + class SnakemakeConfigPlugin(): + + @pytest.fixture() + def model_results(self): + return pd.read_pickle(snakemake.input.model_results) + + return SnakemakeConfigPlugin() + + +if __name__ == "__main__": + run_test(snakemake)