feat(prepro): Add deacon

kubernetes/loculus/templates/loculus-preprocessing-config.yaml

@@ -18,6 +18,14 @@ data:
     {{- if and (hasKey $organismConfig.schema "submissionDataTypes") (hasKey $organismConfig.schema.submissionDataTypes "maxSequencesPerEntry") }}
     max_sequences_per_entry: {{ $organismConfig.schema.submissionDataTypes.maxSequencesPerEntry }}
     {{- end }}
+    {{- if and (hasKey $organismConfig.schema "submissionDataTypes") (hasKey $organismConfig.schema.submissionDataTypes "files") }}
+    {{- if $organismConfig.schema.submissionDataTypes.files.enabled }}
+    submission_file_categories:
+    {{- range $organismConfig.schema.submissionDataTypes.files.categories }}
+      - {{ .name }}
+    {{- end }}
+    {{- end }}
+    {{- end }}
     processing_spec:
       {{- $args := dict "metadata" $metadata "referenceGenomes" $organismConfig.referenceGenomes }}
       {{- include "loculus.preprocessingSpecs" $args | nindent 6 }}

kubernetes/loculus/values.schema.json

@@ -814,6 +814,14 @@
                     "type": "string",
                     "description": "Minimizer used for nextclade sort (if require_nextclade_sort_match or if segments are to be assigned using nextclade sort)"
                   },
+                  "deacon_max_host_proportion": {
+                    "groups": ["nextcladePipelineConfigFile"],
+                    "docsIncludePrefix": false,
+                    "type": "number",
+                    "minimum": 0,
+                    "maximum": 1,
+                    "description": "Maximum proportion of host reads in submitted raw read files, as detected by Deacon."
+                  },
                   "backend_request_timeout_seconds": {
                     "groups": ["nextcladePipelineConfigFile"],
                     "docsIncludePrefix": false,

kubernetes/loculus/values.yaml

@@ -1558,6 +1558,7 @@ defaultOrganismConfig: &defaultOrganismConfig
               - name: singleReference
                 genes: []
         nextclade_dataset_server: https://data.clades.nextstrain.org/v3
+        deacon_max_host_proportion: 0.01
   ingest: &ingest
     image: ghcr.io/loculus-project/ingest
     configFile:

kubernetes/loculus/values_preview_server.yaml

@@ -142,7 +142,7 @@ resources:
       memory: "200Mi"
       cpu: "10m"
     limits:
-      memory: "3Gi"
+      memory: "8Gi"
   minio:
     requests:
       memory: "250Mi"
@@ -156,3 +156,23 @@ defaultResources:
   limits:
     memory: "1Gi"
     cpu: "20m"
+
+# Preview only: run just the west-nile preprocessing pipeline while testing the deacon
+# host-depletion step. Each enabled organism runs its own prepro pod and fetches the
+# ~3.1Gi panhuman index, so disabling the rest keeps startup and memory manageable.
+# Deep-merges `enabled: false` onto the base defaultOrganisms entries.
+defaultOrganisms:
+  cchf:
+    enabled: false
+  cchf-multi-ref:
+    enabled: false
+  dummy-organism:
+    enabled: false
+  dummy-organism-with-files:
+    enabled: false
+  ebola-sudan:
+    enabled: false
+  enteroviruses:
+    enabled: false
+  not-aligned-organism:
+    enabled: false

preprocessing/nextclade/environment.yml

@@ -24,3 +24,4 @@ dependencies:
   - pytest=9.1.1
   - pydantic=2.13.4
   - diamond=2.2.2
+  - deacon=0.15.0

preprocessing/nextclade/src/loculus_preprocessing/backend.py

@@ -17,7 +17,7 @@
 
 from .config import Config
 from .datatypes import (
-    FileIdAndName,
+    FileIdAndNameAndReadUrl,
     FileUploadInfo,
     ProcessedEntry,
     UnprocessedData,
@@ -97,7 +97,10 @@ def parse_ndjson(ndjson_data: str) -> Sequence[UnprocessedEntry]:
         submitted_files = json_object["data"].get("files")
         file_mapping = (
             {
-                category: [FileIdAndName(fileId=f["fileId"], name=f["name"]) for f in files]
+                category: [
+                    FileIdAndNameAndReadUrl(fileId=f["fileId"], name=f["name"], url=f.get("url"))
+                    for f in files
+                ]
                 for category, files in submitted_files.items()
             }
             if submitted_files
@@ -245,6 +248,13 @@ def upload_embl_file_to_presigned_url(
         raise RuntimeError(msg)
 
 
+def download_file(config: Config, url: str, save_path: str) -> None:
+    with requests.get(url, stream=True, timeout=config.backend_request_timeout_seconds) as response:
+        response.raise_for_status()
+        with Path(save_path).open("wb") as f:
+            f.writelines(response.iter_content(chunk_size=1024 * 1024))
+
+
 def download_minimizer(config, save_path):
     if config.minimizer_url:
         url = config.minimizer_url

preprocessing/nextclade/src/loculus_preprocessing/config.py

@@ -10,6 +10,7 @@
 from pydantic import BaseModel, Field, model_validator
 
 from loculus_preprocessing.datatypes import (
+    FileCategory,
     FunctionArgs,
     FunctionInputs,
     FunctionName,
@@ -27,6 +28,7 @@
 NEXTCLADE_PREFIX = "nextclade."
 ASSIGNED_REFERENCE_PREFIX = "ASSIGNED_REFERENCE"
 INTERNAL_INPUT_PREFIXES = (NEXTCLADE_PREFIX, ASSIGNED_REFERENCE_PREFIX)
+DEACON_INDEX = "panhuman-1.k31w15.idx"
 
 
 class EmblInfoMetadataPropertyNames(BaseModel):
@@ -119,7 +121,9 @@ class Config(BaseModel):
     require_nextclade_sort_match: bool = False
     minimizer_url: str | None = None
     diamond_dmnd_url: str | None = None
+    deacon_max_host_proportion: float | None = None
 
+    submission_file_categories: list[str] = Field(default_factory=list)
     create_embl_file: bool = False
     scientific_name: str = "Orthonairovirus haemorrhagiae"
     molecule_type: MoleculeType = MoleculeType.GENOMIC_RNA
@@ -142,6 +146,14 @@ def finalize(self):
         if not self.backend_host:  # Set here so we can use organism
             self.backend_host = f"http://127.0.0.1:8079/{self.organism}"
 
+        if FileCategory.RAW_READS in self.submission_file_categories:
+            if self.deacon_max_host_proportion is None:
+                msg = "deacon_max_host_proportion must be set if raw reads submissions are enabled"
+                raise ValueError(msg)
+            if not (0.0 <= self.deacon_max_host_proportion <= 1.0):
+                msg = "deacon_max_host_proportion must be in [0.0, 1.0]"
+                raise ValueError(msg)
+
         self.processing_order = get_processing_order(self)
 
         return self

preprocessing/nextclade/src/loculus_preprocessing/datatypes.py

@@ -1,7 +1,8 @@
+import json
 from collections.abc import Iterable
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from enum import StrEnum, unique
-from typing import Any, Final
+from typing import Any, Final, Self
 
 AccessionVersion = str
 GeneName = str
@@ -82,9 +83,10 @@ def from_single(cls, name: str, type, message: str):
 
 
 @dataclass
-class FileIdAndName:
+class FileIdAndNameAndReadUrl:
     fileId: str  # noqa: N815
     name: str
+    url: str | None = None
 
 
 @dataclass
@@ -95,7 +97,7 @@ class UnprocessedData:
     submissionId: str  # noqa: N815
     metadata: InputMetadata
     unalignedNucleotideSequences: dict[SequenceName, NucleotideSequence | None]  # noqa: N815
-    files: dict[FileCategory, list[FileIdAndName]] | None
+    files: dict[FileCategory, list[FileIdAndNameAndReadUrl]] | None
 
 
 @dataclass
@@ -111,7 +113,7 @@ class UnprocessedEntry:
 @dataclass
 class UnprocessedAfterNextclade:
     inputMetadata: InputMetadata  # noqa: N815
-    files: dict[FileCategory, list[FileIdAndName]] | None
+    files: dict[FileCategory, list[FileIdAndNameAndReadUrl]] | None
     # Derived metadata produced by Nextclade
     nextcladeMetadata: dict[SequenceName, Any] | None  # noqa: N815
     unalignedNucleotideSequences: dict[SequenceName, NucleotideSequence | None]  # noqa: N815
@@ -127,7 +129,7 @@ class UnprocessedAfterNextclade:
 @dataclass
 class ProcessedData:
     metadata: ProcessedMetadata
-    files: dict[FileCategory, list[FileIdAndName]] | None
+    files: dict[FileCategory, list[FileIdAndNameAndReadUrl]] | None
     unalignedNucleotideSequences: dict[SequenceName, Any]  # noqa: N815
     alignedNucleotideSequences: dict[SequenceName, Any]  # noqa: N815
     nucleotideInsertions: dict[SequenceName, Any]  # noqa: N815
@@ -265,3 +267,19 @@ class MoleculeType(StrEnum):
 class Topology(StrEnum):
     LINEAR = "linear"
     CIRCULAR = "circular"
+
+
+@dataclass
+class DeaconSummary:
+    time: float
+    seqs_in: int
+    seqs_out: int
+    seqs_removed: int
+    seqs_removed_proportion: float
+
+    @classmethod
+    def from_json(cls, json_path: str) -> Self:
+        with open(json_path, encoding="utf-8") as f:
+            data = json.load(f)
+        wanted = {f.name for f in fields(cls)}
+        return cls(**{k: v for k, v in data.items() if k in wanted})