feat(deposition): stop checking for GCAs on NCBI, improve test logging (#6878)

Genbank is no longer syncing assembly GCA accessions from ENA - we know
since a over a year when we were told at the ENA days by ENA, see
pathoplexus/ena-submission#80.

However, we still check for the visibility of all GCAs that we submitted
and are not live on Genbank every 12hours. This is custom code and fills
up the logs doing sth that will never return any new results.

Lets stop checking for this.

This PR also contains a PR that improves logging for deposition tests to
be more selective and informative:
- #6881
That PR was accidentally merged into this branch rather than main.

🚀 Preview: Add `preview` label to enable

---------

Co-authored-by: Cornelius Roemer <cornelius.roemer@gmail.com>

Author: anna-parker

ena-submission/pyproject.toml

@@ -23,16 +23,16 @@ build-backend = "hatchling.build"
 packages = ["src/ena_deposition"]
 
 [tool.pytest.ini_options]
-# Live logging - show INFO+ during execution
-log_cli = true
-log_cli_level = "INFO"  # Show INFO logs during execution
-log_cli_format = "%(asctime)s [%(levelname)8s] %(name)s: %(message)s (%(filename)s:%(lineno)d)"
-log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+# Logs are only printed for failing tests (under "Captured log call"),
+# instead of live-streaming (and interleaving) logs for every test.
+log_level = "INFO"
+log_format = "%(asctime)s [%(levelname)8s] [%(test_name)s] %(name)s: %(message)s (%(filename)s:%(lineno)d)"
+log_date_format = "%Y-%m-%d %H:%M:%S"
 
-# File logging - captures everything for detailed analysis
+# File logging - captures everything for detailed analysis, regardless of pass/fail
 log_file = "tests.log"
 log_file_level = "DEBUG"
-log_file_format = "%(asctime)s [%(levelname)8s] %(name)s: %(message)s (%(filename)s:%(lineno)d)"
+log_file_format = "%(asctime)s [%(levelname)8s] [%(test_name)s] %(name)s: %(message)s (%(filename)s:%(lineno)d)"
 log_file_date_format = "%Y-%m-%d %H:%M:%S"
 
 addopts = [

ena-submission/src/ena_deposition/check_external_visibility.py

@@ -73,64 +73,10 @@ def check_visibility(self, config: Config, accession: str) -> datetime | None:
         return None
 
 
-GcaCacheKey = tuple[str] | tuple[str, str] | tuple[str, str, str]
-
-gca_cache: dict[GcaCacheKey, bool] = {}
-
-
-def _check_and_cache_ncbi_gca(config: Config, path_segments: GcaCacheKey) -> bool:
-    """
-    Helper function to check NCBI for a GCA accession part and cache the result.
-    Returns True if found and caches True, False if not found and caches False.
-    """
-    cache_key = path_segments
-
-    if cache_key in gca_cache:
-        return gca_cache[cache_key]
-
-    url_path = "/".join(path_segments)
-    response = requests.get(
-        f"https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/{url_path}/",
-        allow_redirects=False,
-        timeout=config.ncbi_public_search_timeout_seconds,
-    )
-
-    if response.status_code == HTTPStatus.OK:
-        gca_cache[cache_key] = True
-        return True
-    gca_cache[cache_key] = False
-    return False
-
-
-def check_gca_cached(config: "Config", accession: str) -> datetime | None:
-    """
-    Checks if a GCA accession exists on NCBI by querying NCBI's API, with caching.
-    It attempts to validate parts of the accession from longest to shortest.
-    """
-    _prefix, numbers = accession.split("_")
-    first_three = numbers[:3]
-    second_three = numbers[3:6]
-    third_three = numbers[6:9]
-
-    if not _check_and_cache_ncbi_gca(config, (first_three,)):
-        return None
-
-    if not _check_and_cache_ncbi_gca(config, (first_three, second_three)):
-        return None
-
-    if not _check_and_cache_ncbi_gca(config, (first_three, second_three, third_three)):
-        return None
-
-    return datetime.now(pytz.UTC)
-
-
 class NCBIVisibilityChecker(VisibilityChecker):
     """Checker for NCBI visibility"""
 
     def check_visibility(self, config: Config, accession: str) -> datetime | None:
-        if accession.startswith("GCA"):
-            return check_gca_cached(config, accession)
-
         if accession.startswith("PRJ"):
             path = "bioproject"
         elif accession.startswith("SAM"):
@@ -193,12 +139,6 @@ def check_visibility(self, config: Config, accession: str) -> datetime | None:
         accession_field_name_prefix="gca_accession",
         checker_class=ENAVisibilityChecker,
     ),
-    (EntityType.ASSEMBLY, "ncbi_gca_first_publicly_visible"): ColumnCheckConfig(
-        entry_class=AssemblyTableEntry,
-        visibility_column="ncbi_gca_first_publicly_visible",
-        accession_field_name_prefix="gca_accession",
-        checker_class=NCBIVisibilityChecker,
-    ),
 }
 
 
@@ -348,8 +288,6 @@ def check_and_update_visibility(config: Config, stop_event: threading.Event):
         check_and_update_visibility_all_columns(config, db_engine)
         logger.debug("check_and_update_visibility finished, sleeping for a while")
 
-        gca_cache.clear()
-
         elapsed_time = time.time() - start_time
         if elapsed_time < 60 * config.min_between_publicness_checks:
             wait_time = 60 * config.min_between_publicness_checks - elapsed_time

ena-submission/test/conftest.py

@@ -0,0 +1,23 @@
+import logging
+
+import pytest
+
+_test_context = {"name": "no-test"}
+
+_original_log_record_factory = logging.getLogRecordFactory()
+
+
+def _log_record_factory_with_test_name(*args: object, **kwargs: object) -> logging.LogRecord:
+    record = _original_log_record_factory(*args, **kwargs)
+    record.test_name = _test_context["name"]
+    return record
+
+
+logging.setLogRecordFactory(_log_record_factory_with_test_name)
+
+
+@pytest.fixture(autouse=True)
+def _tag_logs_with_test_name(request: pytest.FixtureRequest):
+    _test_context["name"] = request.node.name
+    yield
+    _test_context["name"] = "no-test"

ena-submission/test/test_ena_submission_integration.py

@@ -653,7 +653,6 @@ class TestFirstPublicUpdate(TestSubmission):
         (EntityType.ASSEMBLY, "ena_nucleotide_first_publicly_visible"): NUCLEOTIDE_CONFIG,
         (EntityType.ASSEMBLY, "ncbi_nucleotide_first_publicly_visible"): NUCLEOTIDE_CONFIG,
         (EntityType.ASSEMBLY, "ena_gca_first_publicly_visible"): GCA_CONFIG,
-        (EntityType.ASSEMBLY, "ncbi_gca_first_publicly_visible"): GCA_CONFIG,
     }
 
     @pytest.mark.parametrize(