Address review: date-parse safety, HHS user-agent and CSV detection, Maine nested links

noderaven · noderaven · commit 65ae38425415 · 2026-06-24T18:06:38.000-07:00
diff --git a/breach_scraper/sources/hhs_ocr.py b/breach_scraper/sources/hhs_ocr.py
@@ -47,7 +47,10 @@ def _normalize_date(value: str) -> str:
     cleaned = _clean_text(value)
     if not cleaned:
         return ""
-    return datetime.strptime(cleaned, "%m/%d/%Y").date().isoformat()
+    try:
+        return datetime.strptime(cleaned, "%m/%d/%Y").date().isoformat()
+    except ValueError:
+        return cleaned
 
 
 def _normalize_count(value: str) -> str:
@@ -217,7 +220,10 @@ def fetch_html(
         export_request = Request(
             url,
             data=data,
-            headers={**_build_headers(), "Content-Type": "application/x-www-form-urlencoded"},
+            headers={
+                **_build_headers(user_agent),
+                "Content-Type": "application/x-www-form-urlencoded",
+            },
         )
         with opener.open(export_request, timeout=timeout) as response:  # nosec B310
             charset = response.headers.get_content_charset() or "utf-8"
@@ -228,7 +234,8 @@ def fetch_html(
 
 
 def _parse_csv_export(text: str) -> list[dict[str, str]]:
-    if not text.lstrip().startswith('"'):
+    cleaned = text.lstrip()
+    if cleaned.startswith("<") or "Covered Entity Type" not in cleaned:
         return []
 
     rows = list(csv.reader(io.StringIO(text)))
diff --git a/breach_scraper/sources/maine_ag.py b/breach_scraper/sources/maine_ag.py
@@ -411,6 +411,8 @@ def handle_endtag(self, tag: str) -> None:
             text = item.text
             if self.item_stack:
                 self.item_stack[-1].add_child_text(text)
+                if item.href and not self.item_stack[-1].href:
+                    self.item_stack[-1].href = item.href
             elif text:
                 self.items.append((self.current_section, text, item.href))
 
diff --git a/breach_scraper/sources/or_doj.py b/breach_scraper/sources/or_doj.py
@@ -48,7 +48,10 @@ def _normalize_single_date(value: str) -> str:
     cleaned = _clean_text(value)
     if not cleaned or cleaned in SENTINEL_DATES:
         return ""
-    return datetime.strptime(cleaned, "%m/%d/%Y").date().isoformat()
+    try:
+        return datetime.strptime(cleaned, "%m/%d/%Y").date().isoformat()
+    except ValueError:
+        return cleaned
 
 
 def _normalize_dateish(value: str) -> str:
@@ -61,8 +64,8 @@ def _normalize_dateish(value: str) -> str:
         if not part:
             continue
 
-        if " - " in part:
-            range_parts = [_normalize_single_date(item) for item in part.split(" - ")]
+        if "-" in part:
+            range_parts = [_normalize_single_date(item) for item in re.split(r"\s*-\s*", part)]
             range_parts = [item for item in range_parts if item]
             if range_parts:
                 normalized_parts.append(" to ".join(range_parts))
diff --git a/tests/test_hhs_ocr.py b/tests/test_hhs_ocr.py
@@ -1,6 +1,6 @@
 import unittest
 
-from breach_scraper.sources.hhs_ocr import parse_breach_table
+from breach_scraper.sources.hhs_ocr import _parse_csv_export, parse_breach_table
 
 HHS_HTML = """
 <html>
@@ -59,5 +59,13 @@ def test_parse_breach_table_handles_csv_export(self) -> None:
         self.assertEqual(records[0]["individuals_affected"], "14,442")
 
 
+class TestHhsCsvDetection(unittest.TestCase):
+    def test_html_is_not_treated_as_csv(self) -> None:
+        self.assertEqual(_parse_csv_export("  <html><body>nope</body></html>"), [])
+
+    def test_csv_without_known_header_is_rejected(self) -> None:
+        self.assertEqual(_parse_csv_export('"a","b"\n"1","2"\n'), [])
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_or_doj.py b/tests/test_or_doj.py
@@ -1,6 +1,6 @@
 import unittest
 
-from breach_scraper.sources.or_doj import parse_breach_table
+from breach_scraper.sources.or_doj import _normalize_dateish, parse_breach_table
 
 OREGON_HTML = """
 <html>
@@ -100,5 +100,13 @@ def test_parse_breach_table_handles_live_table_markup(self) -> None:
         self.assertEqual(records[0]["number_affected"], "139,899")
 
 
+class TestOrDojDateNormalization(unittest.TestCase):
+    def test_tight_range_is_split(self) -> None:
+        self.assertEqual(_normalize_dateish("8/13/2025-8/14/2025"), "2025-08-13 to 2025-08-14")
+
+    def test_invalid_date_returns_original(self) -> None:
+        self.assertEqual(_normalize_dateish("13/45/2025"), "13/45/2025")
+
+
 if __name__ == "__main__":
     unittest.main()
