fix!(prepro): Make host field curatable for INSDC-ingested sequences (#6499)

## BREAKING CHANGE

A prepro function has changed its name for clarity, please change
```
preprocessing:
   function: validate_host
```
to:
```
preprocessing:
   function: resolve_host_taxon_id
```

### Summary 

For INSDC ingested sequences, we currently skip host validation and just
take the `hostTaxonId` as-is, if it's provided.

This was done because we did not want to overwrite INSDC-ingested
information during preprocessing, as it would be a type of curation. If
a hypothetical INSDC sequence has a `hostTaxonId` pointing to organism
A, but a scientific name pointing to organism B, our preprocessing would
overwrite this so both fields point to organism A. While this may be
correct, it would mean that this sequence looks different on our end
versus in INSDC.

However, we're now running into issues where we can't perform desired
curations on INSDC-ingested data, specifically for sequences where a
`hostNameScientific` is present but the `hostTaxonId` is missing. This
PR aims to correct this.

### PR Checklist
- [ ] All necessary documentation has been adapted.
- [ ] The implemented feature is covered by appropriate, automated
tests.
- [x] Any manual testing that has been done is documented (i.e. what
exactly was tested?)
I signed in as superuser and confirmed I could revise/curate a sequence
by editing the host field:
<img width="2958" height="692" alt="image"
src="https://github.com/user-attachments/assets/f2df1a02-614e-4385-b533-d983151ad65c"
/>

🚀 Preview: https://remove-insdc-special-case.loculus.org

---------

Co-authored-by: Anna (Anya) Parker <50943381+anna-parker@users.noreply.github.com>

Author: maverbiest

kubernetes/loculus/values.yaml

@@ -1250,7 +1250,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         noInput: true
         orderOnDetailsPage: 1540
         preprocessing:
-          function: validate_host
+          function: resolve_host_taxon_id
           inputs:
             host: host
             hostNameScientific: hostNameScientific

preprocessing/nextclade/src/loculus_preprocessing/processing_functions.py

@@ -1589,7 +1589,7 @@ def replace_identifier(values, replacement):
         )
 
     @staticmethod
-    def validate_host(
+    def resolve_host_taxon_id(
         input_data: InputMetadata,
         output_field: str,
         input_fields: list[str],
@@ -1605,23 +1605,6 @@ def validate_host(
         return the tax_id of the most generic taxon (i.e., the one that's closest to
         the root of the taxonomy)
         """
-        # for INSDC-ingested sequences we just return the id INSDC gives us (if any)
-        # if we ever want to change this behaviour, we just have to remove this short-circuit,
-        # the rest of the function is set up to validate INSDC-ingested data as well
-        if args["is_insdc_ingest_group"]:
-            tax_id = input_data.get("hostTaxonId")
-            if not isinstance(tax_id, str) or not tax_id.isdigit():
-                return ProcessingResult(
-                    datum=None,
-                    warnings=[],
-                    errors=[],
-                )
-            return ProcessingResult(
-                datum=tax_id,
-                warnings=[],
-                errors=[],
-            )
-
         tax_service = args.get("taxonomy_service_url")
         if not tax_service:
             return missing_taxonomy_service_error(input_fields, output_field)
@@ -1650,10 +1633,10 @@ def validate_host(
 
         try:
             response = taxonomy_cache.get_or_fetch(url)
+            body = response.json()
         except requests.exceptions.RequestException as e:
             return taxonomy_network_error(unvalidated, "validating", e, input_fields, output_field)
 
-        body = response.json()
         if response.status_code != requests.codes.ok:
             # an invalid host organism is a warning for INSDC ingested sequences, but an error for everyone else
             message = ProcessingAnnotation.from_fields(
@@ -1663,7 +1646,9 @@ def validate_host(
                 message=f"Host validation for '{unvalidated}' failed with code {response.status_code}: {body.get('detail', '')}",
             )
             return ProcessingResult(
-                datum=None,
+                datum=unvalidated
+                if args["is_insdc_ingest_group"] and unvalidated.isdigit()
+                else None,
                 warnings=[message] if args["is_insdc_ingest_group"] else [],
                 errors=[message] if not args["is_insdc_ingest_group"] else [],
             )
@@ -1704,47 +1689,42 @@ def scientific_name_from_id(
         input_fields: list[str],
         args: FunctionArgs,
     ) -> ProcessingResult:
-        # for INSDC-ingested sequences we just return the name INSDC gives us (if any)
-        # if we ever want to change this behaviour, we just have to remove this short-circuit,
-        # the rest of the function is set up to validate INSDC-ingested data as well
-        if args["is_insdc_ingest_group"]:
-            return ProcessingResult(
-                datum=input_data.get("hostNameScientific"),
-                warnings=[],
-                errors=[],
-            )
-
         tax_service = args.get("taxonomy_service_url")
         if not tax_service:
             return missing_taxonomy_service_error(input_fields, output_field)
 
         tax_id: str | None = input_data.get("hostTaxonId")
         if not tax_id:
             return ProcessingResult(
-                datum=None,
+                datum=input_data.get("hostNameScientific")
+                if args["is_insdc_ingest_group"]
+                else None,
                 warnings=[],
                 errors=[],
             )
 
         url = f"{tax_service}/taxa/{tax_id}"
         try:
             response = taxonomy_cache.get_or_fetch(url)
+            body = response.json()
         except requests.exceptions.RequestException as e:
             return taxonomy_network_error(tax_id, "validating", e, input_fields, output_field)
 
-        body = response.json()
         if response.status_code != requests.codes.ok:
+            message = f"Could not map '{tax_id}' to scientific name. Code {response.status_code}: {body.get('detail', '')}"
+            logger.warning(message)
+            processing_annotation = ProcessingAnnotation.from_fields(
+                input_fields,
+                [output_field],
+                AnnotationSourceType.METADATA,
+                message=message,
+            )
             return ProcessingResult(
-                datum=None,
-                warnings=[],
-                errors=[
-                    ProcessingAnnotation.from_fields(
-                        input_fields,
-                        [output_field],
-                        AnnotationSourceType.METADATA,
-                        message=f"Internal error: could not map '{tax_id}' to scientific name. Code {response.status_code}: {body.get('detail', '')}",
-                    )
-                ],
+                datum=input_data.get("hostNameScientific")
+                if args["is_insdc_ingest_group"]
+                else None,
+                warnings=[processing_annotation] if args["is_insdc_ingest_group"] else [],
+                errors=[processing_annotation] if not args["is_insdc_ingest_group"] else [],
             )
 
         scientific_name = body.get("scientific_name")
@@ -1790,12 +1770,12 @@ def common_name_from_id(
         url = f"{tax_service}/taxa/{tax_id}?find_common_name=true"
         try:
             response = taxonomy_cache.get_or_fetch(url)
+            body = response.json()
         except requests.exceptions.RequestException as e:
             return taxonomy_network_error(
                 tax_id, "getting common name for", e, input_fields, output_field
             )
 
-        body = response.json()
         if response.status_code != requests.codes.ok:
             return ProcessingResult(
                 datum=None,

preprocessing/nextclade/tests/host_processing_config.yaml

@@ -12,7 +12,7 @@ processing_spec:
       hostTaxonId: processed.hostTaxonId
     args: *preprocessingTaxonomyTestArgs
   hostTaxonId:
-    function: validate_host
+    function: resolve_host_taxon_id
     inputs:
       host: host
       hostNameScientific: hostNameScientific

preprocessing/nextclade/tests/test_host_name_validation.py

@@ -71,7 +71,10 @@ def test_host_processing_direct_submission(mock_session: MagicMock) -> None:
     assert metadata["hostNameCommon"] == "yellow fever mosquito"
     assert result[0].processed_entry.errors == []
 
-    # All three fields should have hit the taxonomy service
+    # Three distinct taxonomy-service URLs are fetched, so no hits in taxonomy_cache:
+    #   1. resolve_host_taxon_id   -> GET /taxa?scientific_name=Aedes+aegypti
+    #   2. scientific_name_from_id -> GET /taxa/7159
+    #   3. common_name_from_id     -> GET /taxa/7159?find_common_name=true
     assert mock_session.get.call_count == 3
 
 
@@ -96,14 +99,21 @@ def test_host_processing_insdc(mock_session: MagicMock) -> None:
     assert metadata["hostNameCommon"] == "yellow fever mosquito"
     assert result[0].processed_entry.errors == []
 
-    # For INSDC, only use the taxonomy service for common name
-    assert mock_session.get.call_count == 1
+    # Only two URLs are fetched: resolve_host_taxon_id and scientific_name_from_id both build
+    # GET /taxa/7159, so the second one is served from taxonomy_cache (no extra call):
+    #   1. resolve_host_taxon_id   -> GET /taxa/7159
+    #   2. scientific_name_from_id -> GET /taxa/7159  -> cache hit, no call
+    #   3. common_name_from_id     -> GET /taxa/7159?find_common_name=true
+    assert mock_session.get.call_count == 2
 
 
 @patch.object(processing_functions.taxonomy_cache, "session")
 def test_host_processing_invalid_hostname(mock_session: MagicMock) -> None:
-    """When hostNameScientific is not found in the taxonomy, hostTaxonId is None,
-    which causes hostNameScientific and hostNameCommon to also fail."""
+    """For an INSDC-ingested sequence whithout a scientific name but with a
+    taxon id, hostTaxonId should be None but hostNameScientific should be set
+    to the provided value.
+    There should also be a warning saying host validation failed
+    """
     mock_session.get.return_value = make_response(404, {"detail": "not found"})
     config = get_config(HOST_PROCESSING_CONFIG, ignore_args=True)
 
@@ -119,9 +129,9 @@ def test_host_processing_invalid_hostname(mock_session: MagicMock) -> None:
     assert metadata["hostNameScientific"] == "not a real species"
     assert metadata["hostNameCommon"] is None
 
-    # For INSDC, nothing should hit the taxonomy service and no warnings are raised
-    assert mock_session.get.call_count == 0
-    assert len(result[0].processed_entry.warnings) == 0
+    assert mock_session.get.call_count == 1
+    assert len(result[0].processed_entry.warnings) == 1
+    assert "Host validation for" in result[0].processed_entry.warnings[0].message
     assert result[0].processed_entry.errors == []