Skip to content

Commit faebb28

Browse files
JoseEspinosaJoseEspinosa
authored
Merge pull request nf-core#322 from Australian-Structural-Biology-Computing/clean-dbs
Refactor database paths
2 parents a69b49b + cbb1713 commit faebb28

13 files changed

Lines changed: 132 additions & 190 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
4444
- [[#263](https://github.com/nf-core/proteinfold/issues/263)] - Removed broken colabfold options (`auto` and `alphafold2`)
4545
- [[PR #316](https://github.com/nf-core/proteinfold/pull/316)] - Add process_gpu label to modules which use GPU.
4646
- [[PR #319](https://github.com/nf-core/proteinfold/pull/319)] - Update boltz workflow to accept YAML as input.
47+
- [[PR #322](https://github.com/nf-core/proteinfold/pull/322)] - Updates and reorganises the reference database directory structure.
4748

4849
### Parameters
4950

‎conf/base.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ process {
5252
withLabel:process_high_memory {
5353
memory = { 200.GB * task.attempt }
5454
}
55-
withLabled:process_gpu {
55+
withLabel:process_gpu {
5656
accelerator = 1
5757
}
5858
withLabel:error_ignore {

‎conf/dbs.config‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
11
/*
22
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3-
Nextflow config file for databases links
3+
Nextflow config file for database links and paths
44
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5-
Defines pointers to the DBS publicly available that store models parametrisations
6-
and data.
5+
Centralizes URLs and local paths for all external resources used in the workflow.
6+
Update version numbers and base directories in one place for maintainability.
77
----------------------------------------------------------------------------------------
88
*/
9+
910
params {
1011

12+
// Dated prefixes, can be modified for alternate versions
13+
alphafold2_params_prefix = "alphafold_params_2022-12-06"
14+
uniref30_prefix = "UniRef30_2023_02"
15+
1116
// AlphaFold2 links
1217
bfd_link = 'https://storage.googleapis.com/alphafold-databases/casp14_versions/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt.tar.gz'
1318
alphafold2_small_bfd_link = 'https://storage.googleapis.com/alphafold-databases/reduced_dbs/bfd-first_non_consensus_sequences.fasta.gz'
@@ -25,9 +30,9 @@ params {
2530
// Alphafold2 paths
2631
bfd_path = "${params.alphafold2_db}/bfd/*"
2732
alphafold2_small_bfd_path = "${params.alphafold2_db}/small_bfd/*"
28-
alphafold2_params_path = "${params.alphafold2_db}/alphafold_params_*/*"
33+
alphafold2_params_path = "${params.alphafold2_db}/params/${params.alphafold2_params_prefix}/*"
2934
alphafold2_mgnify_path = "${params.alphafold2_db}/mgnify/*"
30-
pdb70_path = "${params.alphafold2_db}/pdb70/**"
35+
pdb70_path = "${params.alphafold2_db}/pdb70/*"
3136
alphafold2_pdb_mmcif_path = "${params.alphafold2_db}/pdb_mmcif/mmcif_files"
3237
pdb_obsolete_path = "${params.alphafold2_db}/pdb_mmcif/obsolete.dat"
3338
alphafold2_uniref30_path = "${params.alphafold2_db}/uniref30/*"
@@ -57,16 +62,17 @@ params {
5762
boltz_model_link = 'https://huggingface.co/boltz-community/boltz-1/resolve/main/boltz1_conf.ckpt'
5863

5964
// Boltz paths
60-
boltz_ccd_path = "${params.boltz_db}/ccd.pkl"
61-
boltz_model_path = "${params.boltz_db}/boltz1_conf.ckpt"
65+
boltz_ccd_path = "${params.boltz_db}/params/ccd.pkl"
66+
boltz_model_path = "${params.boltz_db}/params/boltz1_conf.ckpt"
6267

6368
// Colabfold links
6469
colabfold_db_link = 'http://wwwuser.gwdg.de/~compbiol/colabfold/colabfold_envdb_202108.tar.gz'
6570
colabfold_uniref30_link = 'https://wwwuser.gwdg.de/~compbiol/colabfold/uniref30_2302.tar.gz'
6671

6772
// Colabfold paths
68-
colabfold_db_path = "${params.colabfold_db}/colabfold_envdb_202108"
69-
colabfold_uniref30_path = "${params.colabfold_db}/uniref30_2302"
73+
colabfold_db_path = "${params.colabfold_db}/colabfold_envdb/*"
74+
colabfold_uniref30_path = "${params.colabfold_db}/colabfold_uniref30/*"
75+
// Are all these params options needed?
7076
colabfold_alphafold2_params_tags = [
7177
"alphafold2_multimer_v1" : "alphafold_params_colab_2021-10-27",
7278
"alphafold2_multimer_v2" : "alphafold_params_colab_2022-03-02",
@@ -81,10 +87,10 @@ params {
8187
rfaa_paper_weights_link = 'http://files.ipd.uw.edu/pub/RF-All-Atom/weights/RFAA_paper_weights.pt'
8288

8389
// RoseTTAFold_All_Atom paths
84-
uniref30_rosettafold_all_atom_path = "${params.rosettafold_all_atom_db}/uniref30/UniRef30_2020_06/*"
85-
pdb100_rosettafold_all_atom_path = "${params.rosettafold_all_atom_db}/pdb100_2021Mar03/*"
90+
uniref30_rosettafold_all_atom_path = "${params.rosettafold_all_atom_db}/uniref30/*"
91+
pdb100_rosettafold_all_atom_path = "${params.rosettafold_all_atom_db}/pdb100/*"
8692
bfd_rosettafold_all_atom_path = "${params.rosettafold_all_atom_db}/bfd/*"
87-
rfaa_paper_weights_path = "${params.rosettafold_all_atom_db}/RFAA_paper_weights.pt"
93+
rfaa_paper_weights_path = "${params.rosettafold_all_atom_db}/params/RFAA_paper_weights.pt"
8894

8995
// Helixfold3 links
9096
helixfold3_uniclust30_link = 'https://storage.googleapis.com/alphafold-databases/casp14_versions/uniclust30_2018_08_hhsuite.tar.gz'
@@ -102,10 +108,10 @@ params {
102108
helixfold3_obsolete_link = 'https://files.rcsb.org/pub/pdb/data/status/obsolete.dat'
103109

104110
// Helixfold3 paths
105-
helixfold3_uniclust30_path = "${params.helixfold3_db}/uniclust30/*"
106-
helixfold3_ccd_preprocessed_path = "${params.helixfold3_db}/ccd_preprocessed_etkdg.pkl.gz"
107-
helixfold3_rfam_path = "${params.helixfold3_db}/Rfam-14.9_rep_seq.fasta"
108-
helixfold3_init_models_path = "${params.helixfold3_db}/HelixFold3-240814.pdparams"
111+
helixfold3_uniclust30_path = "${params.helixfold3_db}/uniref30/*"
112+
helixfold3_ccd_preprocessed_path = "${params.helixfold3_db}/params/ccd_preprocessed_etkdg.pkl.gz"
113+
helixfold3_rfam_path = "${params.helixfold3_db}/rfam/Rfam-14.9_rep_seq.fasta"
114+
helixfold3_init_models_path = "${params.helixfold3_db}/params/HelixFold3-240814.pdparams"
109115
helixfold3_bfd_path = "${params.helixfold3_db}/bfd/*"
110116
helixfold3_small_bfd_path = "${params.helixfold3_db}/small_bfd/*"
111117
helixfold3_uniprot_path = "${params.helixfold3_db}/uniprot/*"
@@ -122,7 +128,7 @@ params {
122128
esm2_t36_3B_UR50D_contact_regression = 'https://dl.fbaipublicfiles.com/fair-esm/regression/esm2_t36_3B_UR50D-contact-regression.pt'
123129

124130
// Esmfold paths
125-
esmfold_params_path = "${params.esmfold_db}/*"
131+
esmfold_params_path = "${params.esmfold_db}/params/*"
126132

127133
// Foldseek databases paths
128134
foldseek_db = null

‎docs/usage.md‎

Lines changed: 40 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -78,139 +78,48 @@ To provide the predownloaded AlphaFold2 databases and parameters you can specify
7878
<details markdown="1">
7979
<summary>Directory structure</summary>
8080
```console
81-
├── alphafold_params_2022-12-06
82-
│   ├── LICENSE
83-
│   ├── params_model_1_multimer.npz
84-
│   ├── params_model_1_multimer_v2.npz
85-
│   ├── params_model_1_multimer_v3.npz
86-
│   ├── params_model_1.npz
87-
│   ├── params_model_1_ptm.npz
88-
│   ├── params_model_2_multimer.npz
89-
│   ├── params_model_2_multimer_v2.npz
90-
│   ├── params_model_2_multimer_v3.npz
91-
│   ├── params_model_2.npz
92-
│   ├── params_model_2_ptm.npz
93-
│   ├── params_model_3_multimer.npz
94-
│   ├── params_model_3_multimer_v2.npz
95-
│   ├── params_model_3_multimer_v3.npz
96-
│   ├── params_model_3.npz
97-
│   ├── params_model_3_ptm.npz
98-
│   ├── params_model_4_multimer.npz
99-
│   ├── params_model_4_multimer_v2.npz
100-
│   ├── params_model_4_multimer_v3.npz
101-
│   ├── params_model_4.npz
102-
│   ├── params_model_4_ptm.npz
103-
│   ├── params_model_5_multimer.npz
104-
│   ├── params_model_5_multimer_v2.npz
105-
│   ├── params_model_5_multimer_v3.npz
106-
│   ├── params_model_5.npz
107-
│   └── params_model_5_ptm.npz
81+
├── params
82+
│   └── alphafold_params_2022-12-06
83+
│   ├── LICENSE
84+
│   ├── params_model_1_multimer_v3.npz
85+
│   ├── params_model_1.npz
86+
│   ├── params_model_1_ptm.npz
87+
│   ├── params_model_2_multimer_v3.npz
88+
│   ├── params_model_2.npz
89+
│   ├── params_model_2_ptm.npz
90+
│   ├── params_model_3_multimer_v3.npz
91+
│   ├── params_model_3.npz
92+
│   ├── params_model_3_ptm.npz
93+
│   ├── params_model_4_multimer_v3.npz
94+
│   ├── params_model_4.npz
95+
│   ├── params_model_4_ptm.npz
96+
│   ├── params_model_5_multimer_v3.npz
97+
│   ├── params_model_5.npz
98+
│   └── params_model_5_ptm.npz
10899
├── mgnify
109-
│   └── mgy_clusters_2022_05.fa
100+
│   └── mgy_clusters.fa
110101
├── pdb70
111-
│   └── pdb70_from_mmcif_200916
112-
│   ├── md5sum
113-
│   ├── pdb70_a3m.ffdata
114-
│   ├── pdb70_a3m.ffindex
115-
│   ├── pdb70_clu.tsv
116-
│   ├── pdb70_cs219.ffdata
117-
│   ├── pdb70_cs219.ffindex
118-
│   ├── pdb70_hhm.ffdata
119-
│   ├── pdb70_hhm.ffindex
120-
│   └── pdb_filter.dat
102+
│   ├── md5sum
103+
│   ├── pdb70_a3m.ffdata
104+
│   ├── pdb70_a3m.ffindex
105+
│   ├── pdb70_clu.tsv
106+
│   ├── pdb70_cs219.ffdata
107+
│   ├── pdb70_cs219.ffindex
108+
│   ├── pdb70_hhm.ffdata
109+
│   ├── pdb70_hhm.ffindex
110+
│   └── pdb_filter.dat
121111
├── pdb_mmcif
122112
│   ├── mmcif_files
123113
│   │   ├── 1g6g.cif
124114
│   │   ├── 1go4.cif
125115
│   │   ├── 1isn.cif
126-
│   │   ├── 1kuu.cif
127-
│   │   ├── 1m7s.cif
128-
│   │   ├── 1mwq.cif
129-
│   │   ├── 1ni5.cif
130116
│   │   ├── 1qgd.cif
131117
│   │   ├── 1tp9.cif
132-
│   │   ├── 1wa9.cif
133-
│   │   ├── 1ye5.cif
134-
│   │   ├── 1yhl.cif
135-
│   │   ├── 2bjd.cif
136-
│   │   ├── 2bo9.cif
137-
│   │   ├── 2e7t.cif
138-
│   │   ├── 2fyg.cif
139-
│   │   ├── 2j0q.cif
140-
│   │   ├── 2jcq.cif
141-
│   │   ├── 2m4k.cif
142-
│   │   ├── 2n9o.cif
143-
│   │   ├── 2nsx.cif
144-
│   │   ├── 2w4u.cif
145-
│   │   ├── 2wd6.cif
146-
│   │   ├── 2wh5.cif
147-
│   │   ├── 2wji.cif
148-
│   │   ├── 2yu3.cif
149-
│   │   ├── 3cw2.cif
150-
│   │   ├── 3d45.cif
151-
│   │   ├── 3gnz.cif
152-
│   │   ├── 3j0a.cif
153-
│   │   ├── 3jaj.cif
154-
│   │   ├── 3mzo.cif
155-
│   │   ├── 3nrn.cif
156-
│   │   ├── 3piv.cif
157-
│   │   ├── 3pof.cif
158-
│   │   ├── 3pvd.cif
159-
│   │   ├── 3q45.cif
160-
│   │   ├── 3qh6.cif
161-
│   │   ├── 3rg2.cif
162-
│   │   ├── 3sxe.cif
163-
│   │   ├── 3uai.cif
164-
│   │   ├── 3uid.cif
165-
│   │   ├── 3wae.cif
166-
│   │   ├── 3wt1.cif
167-
│   │   ├── 3wtr.cif
168-
│   │   ├── 3wy2.cif
169-
│   │   ├── 3zud.cif
170-
│   │   ├── 4bix.cif
171-
│   │   ├── 4bzx.cif
172-
│   │   ├── 4c1n.cif
173-
│   │   ├── 4cej.cif
174-
│   │   ├── 4chm.cif
175-
│   │   ├── 4fzo.cif
176-
│   │   ├── 4i1f.cif
177-
│   │   ├── 4ioa.cif
178-
│   │   ├── 4j6o.cif
179-
│   │   ├── 4m9q.cif
180-
│   │   ├── 4mal.cif
181-
│   │   ├── 4nhe.cif
182118
│   │   ├── 4o2w.cif
183-
│   │   ├── 4pzo.cif
184-
│   │   ├── 4qlx.cif
185-
│   │   ├── 4uex.cif
186-
│   │   ├── 4zm4.cif
187-
│   │   ├── 4zv1.cif
188-
│   │   ├── 5aj4.cif
189-
│   │   ├── 5frs.cif
190-
│   │   ├── 5hwo.cif
191-
│   │   ├── 5kbk.cif
192-
│   │   ├── 5odq.cif
193-
│   │   ├── 5u5t.cif
194-
│   │   ├── 5wzq.cif
195-
│   │   ├── 5x9z.cif
196-
│   │   ├── 5xe5.cif
197-
│   │   ├── 5ynv.cif
198-
│   │   ├── 5yud.cif
199-
│   │   ├── 5z5c.cif
200-
│   │   ├── 5zb3.cif
201-
│   │   ├── 5zlg.cif
202-
│   │   ├── 6a6i.cif
203-
│   │   ├── 6az3.cif
204-
│   │   ├── 6ban.cif
205-
│   │   ├── 6g1f.cif
206-
│   │   ├── 6ix4.cif
207-
│   │   ├── 6jwp.cif
208-
│   │   ├── 6ng9.cif
209-
│   │   ├── 6ojj.cif
210-
│   │   ├── 6s0x.cif
211119
│   │   ├── 6sg9.cif
212120
│   │   ├── 6vi4.cif
213-
│   │   └── 7sp5.cif
121+
│   │   ├── 7sp5.cif
122+
│   │   └── ...
214123
│   └── obsolete.dat
215124
├── pdb_seqres
216125
│   └── pdb_seqres.txt
@@ -219,12 +128,13 @@ To provide the predownloaded AlphaFold2 databases and parameters you can specify
219128
├── uniprot
220129
│   └── uniprot.fasta
221130
├── uniref30
222-
│   ├── UniRef30_2021_03_a3m.ffdata
223-
│   ├── UniRef30_2021_03_a3m.ffindex
224-
│   ├── UniRef30_2021_03_cs219.ffdata
225-
│   ├── UniRef30_2021_03_cs219.ffindex
226-
| ├── UniRef30_2021_03_hhm.ffdata
227-
│   └── UniRef30_2021_03_hhm.ffindex
131+
│ ├── UniRef30_2023_02_a3m.ffdata
132+
│ ├── UniRef30_2023_02_a3m.ffindex
133+
│ ├── UniRef30_2023_02_cs219.ffdata
134+
│ ├── UniRef30_2023_02_cs219.ffindex
135+
| ├── UniRef30_2023_02_hhm.ffdata
136+
│ ├── UniRef30_2023_02_hhm.ffindex
137+
│ └── UniRef30_2023_02.md5sums
228138
└── uniref90
229139
└── uniref90.fasta
230140
```
@@ -308,7 +218,7 @@ If you specify the `--colabfold_db <PATH>` parameter, the directory structure of
308218
<details markdown="1">
309219
<summary>Directory structure</summary>
310220
```console
311-
├── colabfold_envdb_202108
221+
├── colabfold_envdb
312222
│   ├── colabfold_envdb_202108_db.0
313223
│   ├── colabfold_envdb_202108_db.1
314224
│   ├── colabfold_envdb_202108_db.10
@@ -398,7 +308,7 @@ If you specify the `--colabfold_db <PATH>` parameter, the directory structure of
398308
│   ├── params_model_4.npz
399309
│   ├── params_model_5_multimer_v3.npz
400310
│   └── params_model_5.npz
401-
└── uniref30_2302
311+
└── uniref30
402312
├── uniref30_2302_aln.tsv
403313
├── uniref30_2302_db.0
404314
├── uniref30_2302_db.1
@@ -460,7 +370,7 @@ nextflow run nf-core/proteinfold \
460370
If you specify the `--esmfold_db <PATH>` parameter, the directory structure of your path should be like this:
461371

462372
```console
463-
└── checkpoints
373+
└── params
464374
├── esm2_t36_3B_UR50D-contact-regression.pt
465375
├── esm2_t36_3B_UR50D.pt
466376
└── esmfold_3B_v1.pt

‎main.nf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ if (params.mode.toLowerCase().split(",").contains("helixfold3")) {
4040
}
4141
if (params.mode.toLowerCase().split(",").contains("boltz")) {
4242
include { PREPARE_BOLTZ_DBS } from './subworkflows/local/prepare_boltz_dbs'
43-
include { BOLTZ } from './workflows/boltz'
43+
include { BOLTZ } from './workflows/boltz'
4444
}
4545
if (params.mode.toLowerCase().split(",").contains("colabfold") || params.mode.toLowerCase().split(",").contains("boltz")) {
4646
include { PREPARE_COLABFOLD_DBS } from './subworkflows/local/prepare_colabfold_dbs'

‎modules/local/run_alphafold2/main.nf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,13 @@ process RUN_ALPHAFOLD2 {
4646
error("Local RUN_ALPHAFOLD2 module does not support Conda. Please use Docker / Singularity / Podman instead.")
4747
}
4848
def args = task.ext.args ?: ''
49-
def db_preset_cmd = db_preset ? "full_dbs --bfd_database_path=./bfd/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt --uniref30_database_path=./uniref30/UniRef30_2021_03" :
49+
def db_preset_cmd = db_preset ? "full_dbs --bfd_database_path=./bfd/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt --uniref30_database_path=./uniref30/${params.uniref30_prefix}" :
5050
"reduced_dbs --small_bfd_database_path=./small_bfd/bfd-first_non_consensus_sequences.fasta"
5151
if (alphafold2_model_preset == 'multimer') {
5252
alphafold2_model_preset += " --pdb_seqres_database_path=./pdb_seqres/pdb_seqres.txt --uniprot_database_path=./uniprot/uniprot.fasta "
5353
}
5454
else {
55-
alphafold2_model_preset += " --pdb70_database_path=./pdb70/pdb70_from_mmcif_200916/pdb70 "
55+
alphafold2_model_preset += " --pdb70_database_path=./pdb70/pdb70 "
5656
}
5757
"""
5858
if [ -f pdb_seqres/pdb_seqres.txt ]
@@ -66,7 +66,7 @@ process RUN_ALPHAFOLD2 {
6666
--output_dir=\$PWD \
6767
--data_dir=\$PWD \
6868
--uniref90_database_path=./uniref90/uniref90.fasta \
69-
--mgnify_database_path=./mgnify/mgy_clusters_2022_05.fa \
69+
--mgnify_database_path=./mgnify/mgy_clusters.fa \
7070
--template_mmcif_dir=./mmcif_files \
7171
--obsolete_pdbs_path=./obsolete_pdb/obsolete.dat \
7272
$args

‎modules/local/run_alphafold2_msa/main.nf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ process RUN_ALPHAFOLD2_MSA {
3737
error("Local RUN_ALPHAFOLD2_MSA module does not support Conda. Please use Docker / Singularity / Podman instead.")
3838
}
3939
def args = task.ext.args ?: ''
40-
def db_preset_cmd = db_preset ? "full_dbs --bfd_database_path=./bfd/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt --uniref30_database_path=./uniref30/UniRef30_2021_03" :
40+
def db_preset_cmd = db_preset ? "full_dbs --bfd_database_path=./bfd/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt --uniref30_database_path=./uniref30/${params.uniref30_prefix}" :
4141
"reduced_dbs --small_bfd_database_path=./small_bfd/bfd-first_non_consensus_sequences.fasta"
4242
if (alphafold2_model_preset == 'multimer') {
4343
alphafold2_model_preset += " --pdb_seqres_database_path=./pdb_seqres/pdb_seqres.txt --uniprot_database_path=./uniprot/uniprot.fasta "
4444
}
4545
else {
46-
alphafold2_model_preset += " --pdb70_database_path=./pdb70/pdb70_from_mmcif_200916/pdb70 "
46+
alphafold2_model_preset += " --pdb70_database_path=./pdb70/pdb70 "
4747
}
4848
"""
4949
if [ -f pdb_seqres/pdb_seqres.txt ]
@@ -56,7 +56,7 @@ process RUN_ALPHAFOLD2_MSA {
5656
--output_dir=\$PWD \
5757
--data_dir=\$PWD \
5858
--uniref90_database_path=./uniref90/uniref90.fasta \
59-
--mgnify_database_path=./mgnify/mgy_clusters_2022_05.fa \
59+
--mgnify_database_path=./mgnify/mgy_clusters.fa \
6060
--template_mmcif_dir=./pdb_mmcif/mmcif_files \
6161
--obsolete_pdbs_path=./pdb_mmcif/obsolete.dat \
6262
$args

0 commit comments

Comments
 (0)