Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
e45863a
replace output directory variable with pathvars
ahmig May 3, 2026
c752e29
replace logs directory variable with pathvars
ahmig May 3, 2026
660af2f
ci: update log packing
ahmig May 3, 2026
a7ba5cc
replace report folder variables with pathvars
ahmig May 3, 2026
d79ff45
update .gitignore
ahmig May 3, 2026
df8263e
revert CONTEXT_FASTA to null
ahmig May 3, 2026
7ed14e7
fix dataset pathvar in param
ahmig May 3, 2026
ba009a4
remove support of Snakemake v7
ahmig May 3, 2026
c7e5376
ci: update logs folder
ahmig May 4, 2026
ab4b025
fix: use maximum precision for the time signal JSON
ahmig May 6, 2026
8ef5c15
fix: ensure all key columns are present
ahmig May 17, 2026
b697a7c
fix: fill missing dN and dS count categories with zero
ahmig Jun 4, 2026
7b2831d
fix missing column
ahmig Jun 5, 2026
6d7c81a
fix: set max depth for mpileup
ahmig Jun 12, 2026
8c2a218
fix: strip remove whitespace from parsed version
ahmig Jun 19, 2026
ed29627
refactor: allow extra parameters on VCF fetch rule
ahmig Jun 30, 2026
17cd93f
refactor: follow curl redirects for VCF by default
ahmig Jun 30, 2026
80f53ca
refactor: add amplitude and unique observation filters to pairwise co…
ahmig Jul 8, 2026
ec77665
refactor: update params names for clarity
ahmig Jul 8, 2026
580588a
add param to combine pairwise correlation filter
ahmig Jul 17, 2026
51f1b1b
update if-else pattern
ahmig Jul 17, 2026
f870be4
fix typo
ahmig Jul 17, 2026
8e58449
fail curl on HTTP errors
ahmig Jul 22, 2026
c42e765
update biopython environment
ahmig Jul 22, 2026
4b2b048
fix escape sequences after pandas v3
ahmig Jul 22, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/install.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
with:
channel-priority: strict
activate-environment: snakemake
environment-file: .test/environment_v7.yaml
environment-file: .test/environment_v9.yaml
- name: Create environments
shell: bash -el {0}
run: snakemake --snakefile .test/Snakefile --configfile config/config.yaml .test/targets.yaml --conda-create-envs-only --use-conda -c1 --conda-frontend conda
Expand All @@ -29,7 +29,7 @@ jobs:
with:
channel-priority: strict
activate-environment: snakemake
environment-file: .test/environment_v7.yaml
environment-file: .test/environment_v9.yaml
- name: Dry run
shell: bash -el {0}
run: snakemake --snakefile .test/Snakefile --configfile config/config.yaml .test/targets.yaml --dry-run
4 changes: 2 additions & 2 deletions .github/workflows/test_apptainer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ jobs:
- name: Pack logs
if: success() || failure()
shell: bash -el {0}
run: tar czf logs.tar.gz .test/output .snakemake/log
- name: Upload output file
run: tar czf logs.tar.gz logs .snakemake/log
- name: Upload logs file
if: success() || failure()
uses: actions/upload-artifact@v4
with:
Expand Down
39 changes: 0 additions & 39 deletions .github/workflows/test_v7.yml

This file was deleted.

4 changes: 2 additions & 2 deletions .github/workflows/test_v9.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ jobs:
- name: Pack logs
if: success() || failure()
shell: bash -el {0}
run: tar czf logs.tar.gz .test/output .snakemake/log
- name: Upload output file
run: tar czf logs.tar.gz logs .snakemake/log
- name: Upload logs file
if: success() || failure()
uses: actions/upload-artifact@v4
with:
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

# Analysis files
/data/
/output/
/results/
/logs/

# MAC
.DS_Store
Expand Down
7 changes: 5 additions & 2 deletions .test/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@ from snakemake.utils import min_version
import subprocess


min_version("7.19")
min_version("9.12")

# Workflow version
__version__ = "test"

pathvars:
dataset = config["OUTPUT_NAME"]

# Rules
include: "../workflow/core.smk"
include: "../workflow/rules/demix.smk"
Expand All @@ -19,4 +22,4 @@ include: "../workflow/rules/report.smk"

rule all:
input:
OUTDIR/f"{OUTPUT_NAME}.report.html"
"<results>/<dataset>/report.html"
10 changes: 0 additions & 10 deletions .test/environment_v7.yaml

This file was deleted.

2 changes: 0 additions & 2 deletions .test/targets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ SAMPLES:
fasta: ".test/data/fasta/sample3.fasta"
METADATA:
".test/data/metadata.csv"
OUTPUT_DIRECTORY:
".test/output"
CONTEXT_FASTA:
".test/data/context.fasta"
OUTPUT_NAME:
Expand Down
34 changes: 9 additions & 25 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
FROM condaforge/miniforge3:latest
LABEL io.github.snakemake.containerized="true"
LABEL io.github.snakemake.conda_env_hash="fb2a04b98efa75d8b830a16722a8717eb91a3c66659e3c78b90b0cee3a50db69"

# Step 2: Retrieve conda environments

LABEL io.github.snakemake.conda_env_hash="405c5473ec0f18eae88ec6de97e8b3689fe0fb53cced156e8f6efe589c815a36"
# Conda environment:
# source: workflow/envs/afwdist.yaml
# prefix: /conda-envs/9c24a867826615972cc288081976e7fc
Expand All @@ -14,20 +11,19 @@ LABEL io.github.snakemake.conda_env_hash="fb2a04b98efa75d8b830a16722a8717eb91a3c
# - afwdist==1.0.0
RUN mkdir -p /conda-envs/9c24a867826615972cc288081976e7fc
COPY workflow/envs/afwdist.yaml /conda-envs/9c24a867826615972cc288081976e7fc/environment.yaml

# Conda environment:
# source: workflow/envs/biopython.yaml
# prefix: /conda-envs/162796cecea22d99c8702138f0c48e2f
# prefix: /conda-envs/0744a49115d0593f017b8400a3000df0
# channels:
# - conda-forge
# - bioconda
# - nodefaults
# dependencies:
# - python==3.10
# - biopython==1.81
# - pandas==2.0.3
RUN mkdir -p /conda-envs/162796cecea22d99c8702138f0c48e2f
COPY workflow/envs/biopython.yaml /conda-envs/162796cecea22d99c8702138f0c48e2f/environment.yaml

# - python==3.14.6
# - biopython==1.87
# - pandas==3.0.3
RUN mkdir -p /conda-envs/0744a49115d0593f017b8400a3000df0
COPY workflow/envs/biopython.yaml /conda-envs/0744a49115d0593f017b8400a3000df0/environment.yaml
# Conda environment:
# source: workflow/envs/fetch.yaml
# prefix: /conda-envs/9439457f932a4fbca3665c9ea1ac2f0a
Expand All @@ -39,7 +35,6 @@ COPY workflow/envs/biopython.yaml /conda-envs/162796cecea22d99c8702138f0c48e2f/e
# - curl
RUN mkdir -p /conda-envs/9439457f932a4fbca3665c9ea1ac2f0a
COPY workflow/envs/fetch.yaml /conda-envs/9439457f932a4fbca3665c9ea1ac2f0a/environment.yaml

# Conda environment:
# source: workflow/envs/freyja.yaml
# prefix: /conda-envs/bb4c5f3a509433cc08861582fab4a705
Expand All @@ -50,7 +45,6 @@ COPY workflow/envs/fetch.yaml /conda-envs/9439457f932a4fbca3665c9ea1ac2f0a/envir
# - freyja==2.0.1
RUN mkdir -p /conda-envs/bb4c5f3a509433cc08861582fab4a705
COPY workflow/envs/freyja.yaml /conda-envs/bb4c5f3a509433cc08861582fab4a705/environment.yaml

# Conda environment:
# source: workflow/envs/gisaidr.yaml
# prefix: /conda-envs/3fad3c9cdfa40bee9404f6a2e8fda69f
Expand All @@ -64,7 +58,6 @@ COPY workflow/envs/freyja.yaml /conda-envs/bb4c5f3a509433cc08861582fab4a705/envi
RUN mkdir -p /conda-envs/3fad3c9cdfa40bee9404f6a2e8fda69f
COPY workflow/envs/gisaidr.yaml /conda-envs/3fad3c9cdfa40bee9404f6a2e8fda69f/environment.yaml
COPY workflow/envs/gisaidr.post-deploy.sh /conda-envs/3fad3c9cdfa40bee9404f6a2e8fda69f/post-deploy.sh

# Conda environment:
# source: workflow/envs/iqtree.yaml
# prefix: /conda-envs/0a608afb24723cb6fa8aef748f5efbc8
Expand All @@ -75,7 +68,6 @@ COPY workflow/envs/gisaidr.post-deploy.sh /conda-envs/3fad3c9cdfa40bee9404f6a2e8
# - iqtree==2.2.2.3
RUN mkdir -p /conda-envs/0a608afb24723cb6fa8aef748f5efbc8
COPY workflow/envs/iqtree.yaml /conda-envs/0a608afb24723cb6fa8aef748f5efbc8/environment.yaml

# Conda environment:
# source: workflow/envs/nextalign.yaml
# prefix: /conda-envs/04a3347f94ddf7e21c34bc49e5246076
Expand All @@ -86,7 +78,6 @@ COPY workflow/envs/iqtree.yaml /conda-envs/0a608afb24723cb6fa8aef748f5efbc8/envi
# - nextalign==2.13
RUN mkdir -p /conda-envs/04a3347f94ddf7e21c34bc49e5246076
COPY workflow/envs/nextalign.yaml /conda-envs/04a3347f94ddf7e21c34bc49e5246076/environment.yaml

# Conda environment:
# source: workflow/envs/pangolin.yaml
# prefix: /conda-envs/fb978640cd765c8a63bbcdc01f3a206b
Expand All @@ -99,7 +90,6 @@ COPY workflow/envs/nextalign.yaml /conda-envs/04a3347f94ddf7e21c34bc49e5246076/e
# - setuptools<81
RUN mkdir -p /conda-envs/fb978640cd765c8a63bbcdc01f3a206b
COPY workflow/envs/pangolin.yaml /conda-envs/fb978640cd765c8a63bbcdc01f3a206b/environment.yaml

# Conda environment:
# source: workflow/envs/quarto_render.yaml
# prefix: /conda-envs/96f3c1cec4b3ce5d72f708992272e9c1
Expand All @@ -114,7 +104,6 @@ COPY workflow/envs/pangolin.yaml /conda-envs/fb978640cd765c8a63bbcdc01f3a206b/en
# - r-heatmaply==1.6.0
RUN mkdir -p /conda-envs/96f3c1cec4b3ce5d72f708992272e9c1
COPY workflow/envs/quarto_render.yaml /conda-envs/96f3c1cec4b3ce5d72f708992272e9c1/environment.yaml

# Conda environment:
# source: workflow/envs/renv.yaml
# prefix: /conda-envs/fe892aca096e6b2883923c8755f9ac77
Expand All @@ -136,7 +125,6 @@ COPY workflow/envs/quarto_render.yaml /conda-envs/96f3c1cec4b3ce5d72f708992272e9
# - r-logger==0.2.2
RUN mkdir -p /conda-envs/fe892aca096e6b2883923c8755f9ac77
COPY workflow/envs/renv.yaml /conda-envs/fe892aca096e6b2883923c8755f9ac77/environment.yaml

# Conda environment:
# source: workflow/envs/snpeff.yaml
# prefix: /conda-envs/0adafb79cb1bec58ef4c77bf4cca4f95
Expand All @@ -147,7 +135,6 @@ COPY workflow/envs/renv.yaml /conda-envs/fe892aca096e6b2883923c8755f9ac77/enviro
# - snpsift==5.1d
RUN mkdir -p /conda-envs/0adafb79cb1bec58ef4c77bf4cca4f95
COPY workflow/envs/snpeff.yaml /conda-envs/0adafb79cb1bec58ef4c77bf4cca4f95/environment.yaml

# Conda environment:
# source: workflow/envs/tools.yaml
# prefix: /conda-envs/1f283441022c3c9d97669994a3c5e8bb
Expand All @@ -159,7 +146,6 @@ COPY workflow/envs/snpeff.yaml /conda-envs/0adafb79cb1bec58ef4c77bf4cca4f95/envi
# - bcftools==1.23
RUN mkdir -p /conda-envs/1f283441022c3c9d97669994a3c5e8bb
COPY workflow/envs/tools.yaml /conda-envs/1f283441022c3c9d97669994a3c5e8bb/environment.yaml

# Conda environment:
# source: workflow/envs/var_calling.yaml
# prefix: /conda-envs/81e46c677a6cc0618c93963d57d17d3f
Expand All @@ -173,10 +159,8 @@ COPY workflow/envs/tools.yaml /conda-envs/1f283441022c3c9d97669994a3c5e8bb/envir
RUN mkdir -p /conda-envs/81e46c677a6cc0618c93963d57d17d3f
COPY workflow/envs/var_calling.yaml /conda-envs/81e46c677a6cc0618c93963d57d17d3f/environment.yaml

# Step 3: Generate conda environments

RUN conda env create --prefix /conda-envs/9c24a867826615972cc288081976e7fc --file /conda-envs/9c24a867826615972cc288081976e7fc/environment.yaml && \
conda env create --prefix /conda-envs/162796cecea22d99c8702138f0c48e2f --file /conda-envs/162796cecea22d99c8702138f0c48e2f/environment.yaml && \
conda env create --prefix /conda-envs/0744a49115d0593f017b8400a3000df0 --file /conda-envs/0744a49115d0593f017b8400a3000df0/environment.yaml && \
conda env create --prefix /conda-envs/9439457f932a4fbca3665c9ea1ac2f0a --file /conda-envs/9439457f932a4fbca3665c9ea1ac2f0a/environment.yaml && \
conda env create --prefix /conda-envs/bb4c5f3a509433cc08861582fab4a705 --file /conda-envs/bb4c5f3a509433cc08861582fab4a705/environment.yaml && \
conda env create --prefix /conda-envs/3fad3c9cdfa40bee9404f6a2e8fda69f --file /conda-envs/3fad3c9cdfa40bee9404f6a2e8fda69f/environment.yaml && \
Expand Down
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
[![Zenodo DOI](https://zenodo.org/badge/627797162.svg)](https://doi.org/10.5281/zenodo.15771867)

![Install workflow](https://github.com/PathoGenOmics-Lab/VIPERA/actions/workflows/install.yml/badge.svg)
![Test workflow with Snakemake v7](https://github.com/PathoGenOmics-Lab/VIPERA/actions/workflows/test_v7.yml/badge.svg)
![Test workflow with Snakemake v9](https://github.com/PathoGenOmics-Lab/VIPERA/actions/workflows/test_v9.yml/badge.svg)
![Test workflow with Snakemake v9 and Apptainer](https://github.com/PathoGenOmics-Lab/VIPERA/actions/workflows/test_apptainer.yml/badge.svg)

Expand All @@ -24,7 +23,7 @@ configuring [the inputs and outputs](config/README.md#inputs-and-outputs) and
[the context dataset](config/README.md#automated-construction-of-a-context-dataset):

```shell
snakemake --use-conda --cores 4 # command for Snakemake v7
snakemake --sdm conda --cores 4
```

We provide a simple script that downloads the [data](https://doi.org/10.20350/digitalCSIC/15648) from [our study](https://doi.org/10.1093/ve/veae018)
Expand All @@ -40,8 +39,9 @@ It supports dependency management through either conda or Apptainer/Singularity,
[run modes documentation](config/README.md#run-modes).

We use continuous integration (CI) to automatically verify that all dependencies install correctly
with Snakemake v7.32.4 (see GitHub Action `Install`), and to test that VIPERA runs
successfully with Snakemake v7.32.4 and v9.15.0 using conda (Actions `Test Sm v(7|9)`).
(see GitHub Action `Install`), and to test that VIPERA runs
successfully using conda (Actions `Test Sm v(7|9)`).
We run Snakemake v9.15.0 for these.
We also test a containerized workflow with Snakemake v9.15.0 and Apptainer using a
[remote image](https://hub.docker.com/r/ahmig/vipera) (Action `Test Sm v9 Apptainer`).
This image is automatically updated in every version (Action `Deploy`).
Expand Down
1 change: 0 additions & 1 deletion build_targets.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ def find_file_with_extension(directory: Path, prefix: str, extensions: List[str]
else:
sys.exit(f"ERROR: metadata file '{args.metadata_csv}' does not exist")
targets["OUTPUT_NAME"] = args.output_name
targets["OUTPUT_DIRECTORY"] = "output"
targets["CONTEXT_FASTA"] = None
targets["MAPPING_REFERENCES_FASTA"] = None

Expand Down
10 changes: 3 additions & 7 deletions config/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@ The path to these input files is set in two configuration files in YAML format:
[config.yaml](/config/config.yaml) (for general workflow settings) and
[targets.yaml](/config/targets.yaml) (for specific dataset-related settings).
The latter must be modified by the user to point the `SAMPLES` and `METADATA`
parameters to your data. The `OUTPUT_DIRECTORY` parameter should point to your
desired results directory.
parameters to your data.

The script [`build_targets.py`](/build_targets.py) simplifies the process of creating
the targets configuration file. To run this script, you need to have PyYAML installed. It
Expand All @@ -62,8 +61,6 @@ SAMPLES:
...
METADATA:
"path/to/metadata.csv"
OUTPUT_DIRECTORY:
"output"
CONTEXT_FASTA:
null
MAPPING_REFERENCES_FASTA:
Expand Down Expand Up @@ -258,11 +255,10 @@ should be modified to fit your needs. Read more about Snakemake profiles
[here](https://snakemake.readthedocs.io/en/stable/executing/cli.html#executing-profiles).
To use the profile, install the
[Snakemake executor plugin for SLURM](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html)
and run one of the following commands:
and run the following command:

```shell
snakemake --slurm --profile profile/slurm # Snakemake v7
snakemake --profile profile/slurm # Snakemake v8+
snakemake --profile profile/slurm
```

Additionally, we offer the option of running the workflow within a containerized
Expand Down
2 changes: 0 additions & 2 deletions config/targets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ METADATA:
"data/metadata.csv"
OUTPUT_NAME:
"case_study"
OUTPUT_DIRECTORY:
"output"
CONTEXT_FASTA:
null
MAPPING_REFERENCES_FASTA:
Expand Down
7 changes: 5 additions & 2 deletions workflow/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@ import subprocess
min_version("7.19")

with open(Path(workflow.basedir).parent / "version.txt") as f:
__version__ = f.read()
__version__ = f.read().strip()

containerized: "docker://ahmig/vipera:v" + __version__

configfile: "config/config.yaml"
configfile: "config/targets.yaml"

pathvars:
dataset = config["OUTPUT_NAME"]

include: "core.smk"
include: "rules/context.smk"
include: "rules/demix.smk"
Expand All @@ -22,4 +25,4 @@ include: "rules/report.smk"

rule all:
input:
OUTDIR/f"{OUTPUT_NAME}.report.html"
"<results>/<dataset>/report.html"
13 changes: 0 additions & 13 deletions workflow/core.smk
Original file line number Diff line number Diff line change
@@ -1,18 +1,5 @@
BASE_PATH = Path(workflow.basedir).parent.resolve()

include: "rules/common.smk"

# Outputs
OUTPUT_NAME = config["OUTPUT_NAME"]
OUTDIR = Path(config["OUTPUT_DIRECTORY"])

# Logging
LOGDIR = OUTDIR / "logs"

# Report
REPORT_DIR_PLOTS = Path(OUTDIR/"report/plots")
REPORT_DIR_TABLES = Path(OUTDIR/"report/tables")

include: "rules/fetch.smk"
include: "rules/fasta.smk"
include: "rules/asr.smk"
Expand Down
7 changes: 4 additions & 3 deletions workflow/envs/biopython.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
channels:
- conda-forge
- bioconda
- nodefaults
dependencies:
- python==3.10
- biopython==1.81
- pandas==2.0.3
- python==3.14.6
- biopython==1.87
- pandas==3.0.3
Loading
Loading