feat(prepro): speed up Nextclade metadata lookup (#6453)

## Summary

This draft PR replaces `dpath.get()` in the Nextclade preprocessing
metadata path with a simple direct lookup for the dot-separated paths
used by Loculus metadata mappings. It also removes the now-unused
`dpath` Conda dependency and adds a focused unit test for the lookup
behavior.

## Why

While investigating west-nile preprocessing, the slow phase after
`Nextclade results available` was dominated by per-entry metadata
processing rather than upload or taxonomy calls. The hot path was
`process_single -> get_output_metadata -> add_input_metadata ->
add_nextclade_metadata -> dpath.get`.

The Nextclade result objects can be large, and Loculus only needs simple
nested dictionary paths here after the existing wildcard truncation
step. Direct traversal avoids the heavy general-purpose `dpath` lookup
overhead.

Full notes:
https://gist.github.com/theosanderson-agent/657269613739be0d318f64a08d37bfa9

## Local timing

For a synthetic 100-entry west-nile batch using saved unprocessed data:

- Before: metadata/process phase around `26.348s`
- After this branch: `process_single_total=1.183s`, mean `0.012s`,
median `0.002s`, max `0.145s`
- Same run had `nextclade_enrich=1.884s`

## Validation

- `ruff format --diff .`
- `ruff check --diff .`
- `PYTHONPATH=src /usr/bin/python3.12 -m pytest
tests/test_nextclade_preprocessing.py::test_get_nested_metadata_uses_simple_dot_paths`

🚀 Preview: https://codex-fast-nextclade-meta.loculus.org

Author: theosanderson

preprocessing/nextclade/environment.yml

@@ -6,7 +6,6 @@ channels:
 dependencies:
   - python=3.14.4
   - biopython=1.87
-  - dpath=2.2.0
   - nextclade=3.21.2
   - pip=25.2
   - uv=0.11.8

preprocessing/nextclade/src/loculus_preprocessing/prepro.py

@@ -5,8 +5,6 @@
 from tempfile import TemporaryDirectory
 from typing import Any
 
-import dpath
-
 from .backend import (
     download_diamond_db,
     download_minimizer,
@@ -134,6 +132,17 @@ def truncate_after_wildcard(path: str, separator: str = ".") -> str:
     return path
 
 
+def get_nested_metadata(metadata: dict[str, Any], path: str, separator: str = ".") -> Any | None:
+    value: Any = metadata
+    for part in path.split(separator):
+        if not isinstance(value, dict):
+            return None
+        value = value.get(part)
+        if value is None:
+            return None
+    return value
+
+
 def add_nextclade_metadata(
     spec: ProcessingSpec,
     unprocessed: UnprocessedAfterNextclade,
@@ -169,12 +178,10 @@ def add_nextclade_metadata(
     ):
         return InputData(datum=None)
 
-    raw: str | None = dpath.get(
+    raw: Any | None = get_nested_metadata(
         unprocessed.nextcladeMetadata[sequence_name],
         truncate_after_wildcard(nextclade_path),
-        separator=".",
-        default=None,
-    )  # type: ignore[assignment]
+    )
 
     match nextclade_path:
         case "frameShifts":

preprocessing/nextclade/tests/test_nextclade_preprocessing.py

@@ -32,7 +32,7 @@
     UnprocessedEntry,
 )
 from loculus_preprocessing.embl import create_flatfile, reformat_authors_from_loculus_to_embl_style
-from loculus_preprocessing.prepro import process_all
+from loculus_preprocessing.prepro import get_nested_metadata, process_all
 from loculus_preprocessing.processing_functions import (
     format_frameshift,
     format_stop_codon,
@@ -59,6 +59,28 @@
 LABELED_PRIVATE_MUTATIONS = "tests/labeledPrivateMutations.json"
 
 
+def test_get_nested_metadata_uses_simple_dot_paths():
+    metadata = {
+        "coverage": 0.98,
+        "qc": {"stopCodons": {"totalStopCodons": 0, "stopCodons": []}},
+        "cladeFounderInfo": {
+            "aaMutations": [{"privateSubstitutions": ["NS1:Y35H"]}],
+        },
+    }
+
+    assert get_nested_metadata(metadata, "coverage") == 0.98
+    assert get_nested_metadata(metadata, "qc.stopCodons.totalStopCodons") == 0
+    assert get_nested_metadata(metadata, "qc.stopCodons.stopCodons") == []
+    assert get_nested_metadata(metadata, "cladeFounderInfo.aaMutations") == [
+        {"privateSubstitutions": ["NS1:Y35H"]},
+    ]
+    assert get_nested_metadata(metadata, "qc.missing.total") is None
+    assert get_nested_metadata(metadata, "coverage.value") is None
+
+    metadata_with_zero = {"qc": {"score": 0}}
+    assert get_nested_metadata(metadata_with_zero, "qc.score") == 0
+
+
 def consensus_sequence(
     type: Literal["single"]
     | Literal["ebola-sudan"]