add configurable HTTP timeout

introduce NCRO HTTP latency harness test

Author: dwr-psandhu

dms_datastore/download_ncro.py

@@ -34,7 +34,8 @@
 REQUEST_CHUNK_YEARS = 5
 ALIGN_CHUNKS_TO_YEAR_MODULUS = True
 RETAIN_INVENTORY_HOURS=24
-NCRO_MAX_WORKERS = 2  
+NCRO_MAX_WORKERS = 4
+NCRO_HTTP_TIMEOUT = 60.0  # seconds; increase if inventory downloads time out
 
 mappings = {
     "Water Temperature": "temp",
@@ -109,10 +110,14 @@ def load_inventory():
     dbase = dstore_config.station_dbase()
     dbase = dbase.loc[dbase["agency"].str.contains("ncro"), :]
 
-    with httpx.Client(timeout=60.0) as client:
-        response = client.get(url)
-        response.raise_for_status()
-        data = response.json()
+    with httpx.Client(timeout=NCRO_HTTP_TIMEOUT) as client:
+        try:
+            response = client.get(url)
+            response.raise_for_status()
+            data = response.json()
+        except httpx.ReadTimeout:
+            logger.error(f"NCRO Inventory: Read timeout fetching site list from {url}")
+            raise
     sites = data["return"]["sites"]
     sites_df = pd.DataFrame(sites)  # database of all NCRO sites
     logger.debug(f"NCRO Inventory: Retrieved list of {len(sites_df)} sites from NCRO")
@@ -125,10 +130,14 @@ def load_inventory():
         names = similar_ncro_station_names(origname)
 
         url2 = f"https://wdlhyd.water.ca.gov/hydstra/sites/{','.join(names)}/traces"
-        with httpx.Client(timeout=60.0) as client:
-            response = client.get(url2)
-            response.raise_for_status()
-            data2 = response.json()
+        with httpx.Client(timeout=NCRO_HTTP_TIMEOUT) as client:
+            try:
+                response = client.get(url2)
+                response.raise_for_status()
+                data2 = response.json()
+            except httpx.ReadTimeout:
+                logger.error(f"NCRO Inventory: Read timeout fetching traces for station {origname} from {url2}")
+                raise
 
         # Flatten the JSON
         flattened_data = []
@@ -190,7 +199,9 @@ async def _async_download_trace(client, site, trace, stime, etime):
         except Exception as e:
             logger.debug(f"Exception on attempt {attempt}: " + str(e))
             if attempt == max_attempt:
-                logger.warning("Failed all attempts to download trace for station   " + site + " trace " + trace)
+                logger.warning(
+                    f"Failed all attempts to download trace for station {site} trace {trace} url {url_trace}: {e}"
+                )
                 return None
             else:
                 await asyncio.sleep(

tests/test_ncro_session_get_harness.py

@@ -0,0 +1,35 @@
+import os
+import time
+
+import pytest
+import requests
+
+
+def _env_true(name: str) -> bool:
+    return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"}
+
+
+def test_ncro_sites_session_get_harness():
+    if not _env_true("RUN_NCRO_HTTP_HARNESS"):
+        pytest.skip("Set RUN_NCRO_HTTP_HARNESS=1 to run NCRO HTTP latency harness")
+
+    url = os.getenv("NCRO_HARNESS_URL", "https://wdlhyd.water.ca.gov/hydstra/sites")
+    timeout_seconds = float(os.getenv("NCRO_HARNESS_TIMEOUT", "200"))
+    max_elapsed_seconds = float(os.getenv("NCRO_HARNESS_MAX_SECONDS", "0"))
+
+    session = requests.Session()
+    start = time.perf_counter()
+    response = session.get(url, timeout=timeout_seconds)
+    elapsed = time.perf_counter() - start
+
+    assert response.ok, f"HTTP {response.status_code} from {url}"
+
+    if max_elapsed_seconds > 0:
+        assert (
+            elapsed <= max_elapsed_seconds
+        ), f"session.get({url}) took {elapsed:.2f}s (threshold={max_elapsed_seconds:.2f}s)"
+
+    print(
+        f"NCRO harness: GET {url} completed in {elapsed:.2f}s "
+        f"(status={response.status_code}, timeout={timeout_seconds:.1f}s)"
+    )