Skip to content

Commit f3d863d

Browse files
jmfernandezjmfernandez
committed
Added new sample scripts for STEERS event next December
1 parent 6e4135f commit f3d863d

3 files changed

Lines changed: 245 additions & 0 deletions

File tree

‎sample-work-to-measure/README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Sample work to test `treecript` and other technical metrics collectors
2+
3+
This directory contains a series of bash scripts which automate several
4+
tasks which can be used as sample work to test the different capabilities
5+
of technical metrics collectors like `treecript`.
6+
7+
* [download-and-test-linux-6.6.bash](download-and-test-linux-6.6.bash) is a script
8+
which fetches content compressed with two different algorithms (in this case,
9+
public snapshots of the Linux Kernel source code) to a temporary directory,
10+
and tests the integrity of both archives as well as lists their internal
11+
contents. It assumes both `tar`, `gunzip` and `unxz` are available.
12+
13+
* [download-and-run-nextflow-workflow.bash](download-and-run-nextflow-workflow.bash):
14+
This script fetches into a temporary directory both an old variant
15+
calling workflow written in Nextflow v1, as well as all its
16+
dependencies: a JVM; a capable old nextflow engine;
17+
the inputs; the reference datasets; ... Once all is fetched the layout
18+
is prepared to be able to run the workflow using the inputs and DBs.
19+
It assumes `docker` is properly setup, up and running for the user using
20+
the script.
21+
22+
* [download-NXF-and-amplicon-analysis-pipeline.bash](download-NXF-and-amplicon-analysis-pipeline.bash)
23+
and [run-downloaded-NXF-and-amplicon-analysis-pipeline.bash](run-downloaded-NXF-and-amplicon-analysis-pipeline.bash):
24+
The first bash script fetches to a given input working directory the workflow
25+
[amplicon-analysis-pipeline v6.0.4](https://github.com/EBI-Metagenomics/amplicon-analysis-pipeline/tree/v6.0.4),
26+
as well as all the input, database and software pre-conditions. Once all the
27+
content is fetched, the directories layout expected by the workflow is
28+
prepared.
29+
30+
The second bash script takes as input a working directory already prepared
31+
by the previous script, as well as an output directory. This last one
32+
will be used to store the results computed by the workflow when it uses
33+
the fetched inputs.
34+

‎sample-work-to-measure/download-NXF-and-amplicon-analysis-pipeline.bash‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/bin/bash
2+
3+
if [ $# -ge 0 ] ; then
4+
workdir="$1"
5+
else
6+
workdir="$(mktemp -d --tmpdir treecript_demo_nextflow.XXXXXXXXXX)"
7+
#workdir=/tmp/ATINA/WORKDIR
8+
#mkdir -p "$workdir"
9+
10+
cleanup() {
11+
set +e
12+
# This is needed in order to avoid
13+
# potential "permission denied" messages
14+
chmod -R u+w "${workdir}"
15+
rm -rf "${workdir}"
16+
}
17+
18+
trap cleanup EXIT
19+
fi
20+
21+
downloadDir="${workdir}/downloads"
22+
softDownloadDir="${workdir}/soft-downloads"
23+
mkdir -p "${softDownloadDir}" "${downloadDir}"
24+
25+
jvm=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.16%2B8/OpenJDK11U-jdk-shenandoah_x64_linux_11.0.16_8.tar.gz
26+
nextflow_version=24.04.6
27+
nextflow=https://github.com/nextflow-io/nextflow/releases/download/v${nextflow_version}/nextflow-${nextflow_version}-all
28+
workflow=https://github.com/EBI-Metagenomics/amplicon-analysis-pipeline/archive/f40c51fa51de3553b3c3e52ab72dc942b485a257.zip
29+
30+
#declare -a input_sample=(
31+
# https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/amplicon/sample1_R1.fastq.gz
32+
# https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/amplicon/sample1_R2.fastq.gz
33+
# https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/amplicon/sample2_R1.fastq.gz
34+
# https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/illumina/amplicon/sample2_R2.fastq.gz
35+
#)
36+
declare -a input_sample=(
37+
https://raw.githubusercontent.com/EBI-Metagenomics/pipeline-v5/master/input_examples/amplicon-paired-ERR2237853_1.fastq.gz
38+
https://raw.githubusercontent.com/EBI-Metagenomics/pipeline-v5/master/input_examples/amplicon-paired-ERR2237853_2.fastq.gz
39+
https://raw.githubusercontent.com/EBI-Metagenomics/pipeline-v5/master/input_examples/amplicon-single-ERR1594332.fastq.gz
40+
https://raw.githubusercontent.com/EBI-Metagenomics/pipeline-v5/master/input_examples/wgs-paired-SRR1620013_1_small.fastq.gz
41+
https://raw.githubusercontent.com/EBI-Metagenomics/pipeline-v5/master/input_examples/wgs-paired-SRR1620013_2_small.fastq.gz
42+
)
43+
44+
declare ssu_db=https://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/silva-ssu/silva-ssu_138.1.tar.gz
45+
declare lsu_db=https://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/silva-lsu/silva-lsu_138.1.tar.gz
46+
declare unite_db=https://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/unite/unite_9.0.tar.gz
47+
declare itsone_db=https://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/itsonedb/itsonedb_1.141.tar.gz
48+
declare pr2_db=https://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/pr2/pr2_5.0.0.tar.gz
49+
declare rrnas_rfam=https://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/rfam/rfam_14.10.tar.gz
50+
declare genome=R64-1-1
51+
declare genome_source=Ensembl
52+
declare organism=Saccharomyces_cerevisiae
53+
declare igenomes=http://igenomes.illumina.com.s3-website-us-east-1.amazonaws.com/${organism}/Ensembl/${genome}/${organism}_${genome_source}_${genome}.tar.gz
54+
55+
echo Fetching jvm, nextflow and workflow
56+
wget -c -nv -P "${softDownloadDir}" "${jvm}" "${nextflow}" "${workflow}"
57+
58+
echo Sleeping 2 seconds
59+
sleep 2
60+
61+
echo Fetching input and reference files
62+
wget -c -nv -P "${downloadDir}" "${input_sample[@]}" "${ssu_db}" "${lsu_db}" "${unite_db}" "${itsone_db}" "${pr2_db}" "${rrnas_rfam}" "${igenomes}"
63+
64+
NXF_HOME="${workdir}/.nextflow"
65+
export NXF_HOME
66+
67+
echo Sleeping 2 seconds
68+
sleep 2
69+
70+
echo Preparing the layout of the reference files
71+
mkdir -p "${downloadDir}/ssu_db"
72+
tar -x -C "${downloadDir}/ssu_db" -f "${downloadDir}"/silva-ssu_*.tar.gz
73+
mkdir -p "${downloadDir}/lsu_db"
74+
tar -x -C "${downloadDir}/lsu_db" -f "${downloadDir}"/silva-lsu_*.tar.gz
75+
mkdir -p "${downloadDir}/unite_db"
76+
tar -x -C "${downloadDir}/unite_db" -f "${downloadDir}"/unite_*.tar.gz
77+
mkdir -p "${downloadDir}/itsone_db"
78+
tar -x -C "${downloadDir}/itsone_db" -f "${downloadDir}"/itsonedb_*.tar.gz
79+
mkdir -p "${downloadDir}/pr2_db"
80+
tar -x -C "${downloadDir}/pr2_db" -f "${downloadDir}"/pr2_*.tar.gz
81+
mkdir -p "${downloadDir}/rrnas_rfam"
82+
tar -x -C "${downloadDir}/rrnas_rfam" -f "${downloadDir}"/rfam_*.tar.gz
83+
mkdir -p "${downloadDir}/igenomes_base"
84+
tar -x -C "${downloadDir}/igenomes_base" -f "${downloadDir}"/${organism}_${genome_source}_${genome}.tar.gz
85+
86+
87+
echo Sleeping 2 seconds
88+
sleep 2
89+
90+
echo Preparing the layout of software and workflow
91+
softDir="${workdir}/soft"
92+
93+
if [ ! -d "${softDir}" ] ; then
94+
mkdir -p "${softDir}"
95+
96+
echo Install JVM
97+
tar -x -C "${softDir}" -f "${softDownloadDir}"/OpenJDK*
98+
mv "${softDir}"/openjdk-*/* "${softDir}"
99+
100+
echo Install Nextflow
101+
cp -p "${softDownloadDir}"/nextflow* "${softDir}"/bin/nextflow
102+
chmod +x "${softDir}"/bin/nextflow*
103+
104+
echo Extract workflow
105+
unzip -d "${workdir}" "${softDownloadDir}"/*.zip
106+
fi

‎sample-work-to-measure/run-downloaded-NXF-and-amplicon-analysis-pipeline.bash‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/bash
2+
3+
scriptDir="$(dirname "$(realpath "$0")")"
4+
5+
if [ $# -ge 1 ] ; then
6+
workdir="$1"
7+
outputdir="$2"
8+
else
9+
echo "Usage: $0 {workdir} {outputdir}"
10+
exit 1
11+
fi
12+
13+
if [ ! -d "$workdir" ] ; then
14+
"$SHELL" "$scriptDir"/download-NXF-and-amplicon-analysis-pipeline.bash "$workdir"
15+
fi
16+
17+
downloadDir="${workdir}/downloads"
18+
softDir="${workdir}/soft"
19+
20+
JAVA_HOME="${softDir}"
21+
export JAVA_HOME
22+
PATH="${softDir}/bin:$PATH"
23+
export PATH
24+
25+
workflowDir="$(echo "${workdir}"/amplicon-analysis-pipeline-*)"
26+
27+
declare genome=R64-1-1
28+
declare genome_source=Ensembl
29+
declare organism=Saccharomyces_cerevisiae
30+
declare igenomes_base="${downloadDir}"/igenomes_base
31+
32+
declare igenomes_prefix="${igenomes_base}"/${organism}/${genome_source}/${genome}
33+
34+
echo Creating input and yaml run files
35+
#cat > "${workdir}"/inputs.csv <<EOF
36+
#sample,fastq_1,fastq_2,single_end
37+
#SAMPLE1_PE,${downloadDir}/sample1_R1.fastq.gz,${downloadDir}/sample1_R2.fastq.gz,false
38+
#SAMPLE2_PE,${downloadDir}/sample2_R1.fastq.gz,${downloadDir}/sample2_R2.fastq.gz,false
39+
#EOF
40+
# SAMPLE3_SE,${downloadDir}/sample1_R1.fastq.gz,,true
41+
# SAMPLE3_SE,${downloadDir}/sample2_R1.fastq.gz,,true
42+
cat > "${workdir}"/inputs.csv <<EOF
43+
sample,fastq_1,fastq_2,single_end
44+
ERR2237853,${downloadDir}/amplicon-paired-ERR2237853_1.fastq.gz,${downloadDir}/amplicon-paired-ERR2237853_2.fastq.gz,false
45+
ERR1594332,${downloadDir}/amplicon-single-ERR1594332.fastq.gz,,true
46+
SRR1620013,${downloadDir}/wgs-paired-SRR1620013_1_small.fastq.gz,${downloadDir}/wgs-paired-SRR1620013_2_small.fastq.gz,false
47+
EOF
48+
49+
cat > "${workdir}"/input_params.yml <<EOF
50+
input: ${workdir}/inputs.csv
51+
52+
ssu_db_fasta: ${downloadDir}/ssu_db/SILVA-SSU.fasta
53+
ssu_db_tax: ${downloadDir}/ssu_db/SILVA-SSU-tax.txt
54+
ssu_db_otu: ${downloadDir}/ssu_db/SILVA-SSU.otu
55+
ssu_db_mscluster: ${downloadDir}/ssu_db/SILVA-SSU.fasta.mscluster
56+
57+
lsu_db_fasta: ${downloadDir}/lsu_db/SILVA-LSU.fasta
58+
lsu_db_tax: ${downloadDir}/lsu_db/SILVA-LSU-tax.txt
59+
lsu_db_otu: ${downloadDir}/lsu_db/SILVA-LSU.otu
60+
lsu_db_mscluster: ${downloadDir}/lsu_db/SILVA-LSU.fasta.mscluster
61+
62+
unite_db_fasta: ${downloadDir}/unite_db/UNITE.fasta
63+
unite_db_tax: ${downloadDir}/unite_db/UNITE-tax.txt
64+
unite_db_otu: ${downloadDir}/unite_db/UNITE.otu
65+
unite_db_mscluster: ${downloadDir}/unite_db/UNITE.fasta.mscluster
66+
67+
itsone_db_fasta: ${downloadDir}/itsone_db/ITSone.fasta
68+
itsone_db_tax: ${downloadDir}/itsone_db/ITSone-tax.txt
69+
itsone_db_otu: ${downloadDir}/itsone_db/ITSone.otu
70+
itsone_db_mscluster: ${downloadDir}/itsone_db/ITSone.fasta.mscluster
71+
72+
pr2_db_fasta: ${downloadDir}/pr2_db/PR2.fasta
73+
pr2_db_tax: ${downloadDir}/pr2_db/PR2-tax.txt
74+
pr2_db_otu: ${downloadDir}/pr2_db/PR2.otu
75+
pr2_db_mscluster: ${downloadDir}/pr2_db/PR2.fasta.mscluster
76+
77+
rrnas_rfam_covariance_model: ${downloadDir}/rrnas_rfam/
78+
rrnas_rfam_claninfo: ${downloadDir}/rrnas_rfam/ribo.clan_info
79+
80+
genome: ${genome}
81+
igenomes_base: ${igenomes_base}
82+
genomes:
83+
R64-1-1:
84+
fasta: ${igenomes_prefix}/Sequence/WholeGenomeFasta/genome.fa
85+
bwa: ${igenomes_prefix}/Sequence/BWAIndex/version0.6.0/
86+
bowtie2: ${igenomes_prefix}/Sequence/Bowtie2Index/
87+
star: ${igenomes_prefix}/Sequence/STARIndex/
88+
bismark: ${igenomes_prefix}/Sequence/BismarkIndex/
89+
gtf: ${igenomes_prefix}/Annotation/Genes/genes.gtf
90+
bed12: ${igenomes_prefix}/Annotation/Genes/genes.bed
91+
92+
validationSchemaIgnoreParams: genomes
93+
94+
outdir: ${outputdir}
95+
EOF
96+
97+
cat > "${workdir}"/custom.conf <<EOF
98+
docker {
99+
enabled = true
100+
autoMounts = true
101+
registry = "quay.io"
102+
}
103+
EOF
104+
105+
nextflow run "${workflowDir}" -c "${workdir}"/custom.conf -params-file "${workdir}"/input_params.yml

0 commit comments

Comments
 (0)