feat!(ingest): Add host field for INSDC sequences during ingest (#6534)

resolves #6533
#6295

This PR modifies ingest to ensure host organism information is processed
uniformly for INSDC-ingested sequences and direct submissions. Because
of this, we can also remove INSDC specific behaviour from preprocessing.
This will be done in a separate PR once we know this ingest change is
correct and stable.

Concretely, ingest now collapses submissions with `hostTaxonId` and/or
`hostNameScientific` into a single `host` field (removing `hostTaxonId`
and`hostNameScientific`). The `hostNameCommon` field is also removed.

## Breaking changes
Not strictly breaking, but when rolling out this PR, this DB surgery
needs to be run to make sure existing INSDC submissions fit the new
model:

<details>
<summary>An old command with a WHERE clause that was too strict
</summary>

The command to run:

```sql
UPDATE sequence_entries
SET unprocessed_data = jsonb_set(
    unprocessed_data,
    '{metadata}',
    ((unprocessed_data -> 'metadata') - 'hostTaxonId' - 'hostNameScientific' - 'hostNameCommon')
    || CASE
         WHEN COALESCE(
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostTaxonId', ''),
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostNameScientific', '')
              ) IS NOT NULL
         THEN jsonb_build_object('host', COALESCE(
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostTaxonId', ''),
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostNameScientific', '')))
         ELSE '{}'::jsonb
       END
)
WHERE unprocessed_data IS NOT NULL
  AND (unprocessed_data -> 'metadata' ? 'hostTaxonId'
       OR unprocessed_data -> 'metadata' ? 'hostNameScientific'
       OR unprocessed_data -> 'metadata' ? 'hostNameCommon')
  AND NOT (unprocessed_data -> 'metadata' ? 'host');
```
</details>



```sql
UPDATE sequence_entries
SET unprocessed_data = jsonb_set(
    unprocessed_data,
    '{metadata}',
    ((unprocessed_data -> 'metadata') - 'hostTaxonId' - 'hostNameScientific' - 'hostNameCommon')
    || CASE
         WHEN COALESCE(
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostTaxonId', ''),
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostNameScientific', '')
              ) IS NOT NULL
         THEN jsonb_build_object('host', COALESCE(
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostTaxonId', ''),
                NULLIF(unprocessed_data -> 'metadata' ->> 'hostNameScientific', '')))
         ELSE '{}'::jsonb
       END
)
WHERE unprocessed_data IS NOT NULL
  AND (unprocessed_data -> 'metadata' ? 'hostTaxonId'
       OR unprocessed_data -> 'metadata' ? 'hostNameScientific'
       OR unprocessed_data -> 'metadata' ? 'hostNameCommon')
  AND NULLIF(unprocessed_data -> 'metadata' ->> 'host', '') IS NULL;
```


### Screenshot

### PR Checklist
- [x] All necessary documentation has been adapted.
- [x] The implemented feature is covered by appropriate, automated
tests.
- [x] Any manual testing that has been done is documented (i.e. what
exactly was tested?)

🚀 Preview: Add `preview` label to enable

---------

Co-authored-by: anna-parker <50943381+anna-parker@users.noreply.github.com>
Co-authored-by: GitHub Action <action@github.com>

Author: 3 people

ingest/scripts/heuristic_group_segments.py

@@ -27,6 +27,7 @@
 from typing import Final
 
 import click
+from prepare_metadata import resolve_host_information
 import orjsonl
 import yaml
 
@@ -267,6 +268,8 @@ def main(
             json.dumps(filtered_record, sort_keys=True).encode(), usedforsecurity=False
         ).hexdigest()
 
+        row = resolve_host_information(row)
+
         orjsonl.append(output_metadata, {"id": joint_key, "metadata": row})
         count += 1
 

ingest/scripts/override_group_segments.py

@@ -35,6 +35,7 @@
 from typing import Any, Final
 
 import click
+from prepare_metadata import resolve_host_information
 import orjsonl  # type: ignore
 import requests
 import yaml
@@ -170,7 +171,7 @@ def get_metadata_of_group(
         json.dumps(filtered_record, sort_keys=True).encode(), usedforsecurity=False
     ).hexdigest()
 
-    return grouped_metadata
+    return resolve_host_information(grouped_metadata)
 
 
 def group_records(
@@ -236,7 +237,9 @@ def write_grouped_metadata(
                 prev_acc = f"{acc}.{prev_ver}"
                 if prev_acc in groups.accession_to_group:
                     group = groups.accession_to_group[prev_acc]
-                    logger.warning(f"Matched {full_accession} to group via previous version {prev_acc}")
+                    logger.warning(
+                        f"Matched {full_accession} to group via previous version {prev_acc}"
+                    )
                     break
 
         if group is None:
@@ -334,13 +337,13 @@ def get_groups_object(groups_json_path: str) -> Groups:
     "--groups",
     required=True,
     type=click.Path(exists=True),
-    help="Path to the JSON file containing the map from group names to lists of accessions."
+    help="Path to the JSON file containing the map from group names to lists of accessions.",
 )
 @click.option(
     "--input-seq",
     required=True,
     type=click.Path(exists=True),
-    help="Path to the JSONL file of input data."
+    help="Path to the JSONL file of input data.",
 )
 @click.option("--input-metadata", required=True, type=click.Path(exists=True))
 @click.option("--output-seq", required=True, type=click.Path())
@@ -355,7 +358,7 @@ def get_groups_object(groups_json_path: str) -> Groups:
 @click.option(
     "--match-previous-accession-versions/--no-match-previous-accession-versions",
     default=False,
-    help="Whether to match against previous versions of accessions (e.g., XX123.1 when XX123.2 is provided)"
+    help="Whether to match against previous versions of accessions (e.g., XX123.1 when XX123.2 is provided)",
 )
 def main(  # noqa: PLR0913, PLR0917
     config_file: str,

ingest/scripts/prepare_metadata.py

@@ -36,6 +36,25 @@ class Config:
     segmented: bool
 
 
+def resolve_host_information(record: dict[str, str]) -> dict[str, str]:
+    """Create a new host field and populate it from hostTaxonId or
+    hostNameScientific (falling back to the empty string) to be consistent
+    with how direct submissions specify the host organism. Any existing
+    hostTaxonId, hostNameScientific, and hostNameCommon fields on the
+    record will be removed.
+
+    This should be done after computing the hash for a record to not trigger
+    revisions for all INSDC data
+    """
+    host = record.get("hostTaxonId") or record.get("hostNameScientific")
+    record.pop("hostTaxonId", None)
+    record.pop("hostNameScientific", None)
+    record.pop("hostNameCommon", None)
+    record["host"] = host
+
+    return record
+
+
 @click.command()
 @click.option("--config-file", required=True, type=click.Path(exists=True))
 @click.option("--input", required=True, type=click.Path(exists=True))
@@ -151,6 +170,11 @@ def main(
 
         record["hash"] = hashlib.md5(prehash.encode(), usedforsecurity=False).hexdigest()
 
+        # for segmented organisms, this has to happen in `heuristic_group_segments.py`
+        # and `override_group_segments.py`
+        if not config.segmented:
+            record = resolve_host_information(record)
+
         orjsonl.append(output, {"id": record[fasta_id_field], "metadata": record})
 
     logger.info(f"Saved metadata for {len(metadata)} sequences")

ingest/tests/expected_output_cchf/revise_metadata.tsv

@@ -1,2 +1,2 @@
-authorAffiliations	authors	bioprojectAccession	biosampleAccession	geoLocAdmin1	geoLocCountry	hostNameScientific	hostTaxonId	isLabHost	ncbiReleaseDate	ncbiSourceDb	ncbiVirusName	ncbiVirusTaxId	sampleCollectionDate	specimenCollectorSampleId	ncbiUpdateDate_L	ncbiUpdateDate_M	ncbiUpdateDate_S	insdcVersion_L	insdcVersion_M	insdcVersion_S	insdcRawReadsAccession_L	insdcRawReadsAccession_M	insdcRawReadsAccession_S	hash_L	hash_M	hash_S	insdcAccessionBase_L	insdcAccessionBase_M	insdcAccessionBase_S	insdcAccessionFull_L	insdcAccessionFull_M	insdcAccessionFull_S	id	hash	accession	fastaIds
-Public Health England, Research	Deryabin, ; Atshabar, B.; Sansyzbaev, Y.; Berezin, V.; Nurmakhanov, T.; Yeskhojayev, O.; Vilkova, A.; Shevtsov, A.; Hewson, R.; Atkinson, B.			Sairam district	Kazakhstan	Hyalomma anatolicum	176092		2016-04-30T00:00:00Z	GenBank	Orthonairovirus haemorrhagiae	3052518	2015	tick pool #134			2016-04-30T00:00:00Z			1						4f8a46f4b233b3d9f05d5336b8b711aa			KX096703			KX096703.1	KX096703.1.S	c7d098a014a36e8b8f87c73621b7d6fc	LOC_0000VXA	KX096703.1.S_S
+authorAffiliations	authors	bioprojectAccession	biosampleAccession	geoLocAdmin1	geoLocCountry	host	isLabHost	ncbiReleaseDate	ncbiSourceDb	ncbiVirusName	ncbiVirusTaxId	sampleCollectionDate	specimenCollectorSampleId	ncbiUpdateDate_L	ncbiUpdateDate_M	ncbiUpdateDate_S	insdcVersion_L	insdcVersion_M	insdcVersion_S	insdcRawReadsAccession_L	insdcRawReadsAccession_M	insdcRawReadsAccession_S	hash_L	hash_M	hash_S	insdcAccessionBase_L	insdcAccessionBase_M	insdcAccessionBase_S	insdcAccessionFull_L	insdcAccessionFull_M	insdcAccessionFull_S	id	hash	accession	fastaIds
+"Public Health England, Research"	"Deryabin, ; Atshabar, B.; Sansyzbaev, Y.; Berezin, V.; Nurmakhanov, T.; Yeskhojayev, O.; Vilkova, A.; Shevtsov, A.; Hewson, R.; Atkinson, B."			Sairam district	Kazakhstan	176092		2016-04-30T00:00:00Z	GenBank	Orthonairovirus haemorrhagiae	3052518	2015	tick pool #134			2016-04-30T00:00:00Z			1						4f8a46f4b233b3d9f05d5336b8b711aa			KX096703			KX096703.1	KX096703.1.S	c7d098a014a36e8b8f87c73621b7d6fc	LOC_0000VXA	KX096703.1.S_S

ingest/tests/expected_output_cchf/submit_metadata.tsv

@@ -1,2 +1,2 @@
-authorAffiliations	authors	bioprojectAccession	biosampleAccession	geoLocAdmin1	geoLocCountry	hostNameScientific	hostTaxonId	isLabHost	ncbiReleaseDate	ncbiSourceDb	ncbiVirusName	ncbiVirusTaxId	sampleCollectionDate	specimenCollectorSampleId	ncbiUpdateDate_L	ncbiUpdateDate_M	ncbiUpdateDate_S	insdcVersion_L	insdcVersion_M	insdcVersion_S	insdcRawReadsAccession_L	insdcRawReadsAccession_M	insdcRawReadsAccession_S	hash_L	hash_M	hash_S	insdcAccessionBase_L	insdcAccessionBase_M	insdcAccessionBase_S	insdcAccessionFull_L	insdcAccessionFull_M	insdcAccessionFull_S	id	hash	fastaIds
-Chumakov Institute of Poliomyelitis and Viral Encephalitides	Lukashev, A. N.; Klimentov, A. S.; Smirnova, S. E.; Dzagurova, T. K.; Drexler, J. F.; Gmyl, A. P.				Uganda	Homo sapiens	9606		2016-12-07T00:00:00Z	GenBank	Orthonairovirus haemorrhagiae	3052518	1958	Nakiwogo	2016-12-07T00:00:00Z		2016-12-07T00:00:00Z	1		1				7b10a4e21daa8a2e693958761be17d53		70954bc35782b5592858ac3f1a6bbf89	KX013483		KX013485	KX013483.1		KX013485.1	KX013483.1.L/KX013485.1.S	bb3a6d8df47cb2891e7b60030a40c335	KX013483.1.L/KX013485.1.S_L KX013483.1.L/KX013485.1.S_S
+authorAffiliations	authors	bioprojectAccession	biosampleAccession	geoLocAdmin1	geoLocCountry	host	isLabHost	ncbiReleaseDate	ncbiSourceDb	ncbiVirusName	ncbiVirusTaxId	sampleCollectionDate	specimenCollectorSampleId	ncbiUpdateDate_L	ncbiUpdateDate_M	ncbiUpdateDate_S	insdcVersion_L	insdcVersion_M	insdcVersion_S	insdcRawReadsAccession_L	insdcRawReadsAccession_M	insdcRawReadsAccession_S	hash_L	hash_M	hash_S	insdcAccessionBase_L	insdcAccessionBase_M	insdcAccessionBase_S	insdcAccessionFull_L	insdcAccessionFull_M	insdcAccessionFull_S	id	hash	fastaIds
+Chumakov Institute of Poliomyelitis and Viral Encephalitides	"Lukashev, A. N.; Klimentov, A. S.; Smirnova, S. E.; Dzagurova, T. K.; Drexler, J. F.; Gmyl, A. P."				Uganda	9606		2016-12-07T00:00:00Z	GenBank	Orthonairovirus haemorrhagiae	3052518	1958	Nakiwogo	2016-12-07T00:00:00Z		2016-12-07T00:00:00Z	1		1				7b10a4e21daa8a2e693958761be17d53		70954bc35782b5592858ac3f1a6bbf89	KX013483		KX013485	KX013483.1		KX013485.1	KX013483.1.L/KX013485.1.S	bb3a6d8df47cb2891e7b60030a40c335	KX013483.1.L/KX013485.1.S_L KX013483.1.L/KX013485.1.S_S