Skip to content

Commit b16aaf0

Browse files
davidhuserdavidhuser
committed
handle ssl cert errors
1 parent 6de73af commit b16aaf0

3 files changed

Lines changed: 221 additions & 15 deletions

File tree

‎overrides.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
"6197": {"domain": "loetschen.ch", "reason": "Shared Lötschental admin"},
7777
"6205": {"domain": "gemeinde-bettmeralp.ch", "reason": "as on website"},
7878
"6285": {"domain": "graechen.ch", "reason": "Wikidata URL incorrect (gemeinde-graechen.ch has SSL issues); actual website at gemeinde.graechen.ch, email on graechen.ch (MS365)"},
79+
"6289": {"domain": "3908.ch", "reason": "SSL cert mismatch on gemeinde-saas-balen.ch (*.metanet.ch cert); website redirects to 3908.ch"},
7980
"6404": {"domain": "ne.ch", "reason": "Using cantonal mail"},
8081
"6408": {"domain": "ne.ch", "reason": "Using cantonal mail"},
8182
"6413": {"domain": "ne.ch", "reason": "Using cantonal mail"},

‎src/mail_sovereignty/resolve.py‎

Lines changed: 71 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import asyncio
22
import json
33
import re
4+
import ssl
45
import time
6+
import warnings
57
from pathlib import Path
68
from typing import Any
79
from urllib.parse import urlparse
@@ -382,17 +384,20 @@ def extract_email_domains(html: str) -> set[str]:
382384
"""Extract email domains from HTML, including TYPO3-obfuscated emails."""
383385
domains = set()
384386

387+
# simple @ in body
385388
for email in EMAIL_RE.findall(html):
386389
domain = email.split("@")[1].lower()
387390
if domain not in SKIP_DOMAINS:
388391
domains.add(domain)
389392

393+
# mailto:
390394
for email in re.findall(r'mailto:([^">\s?]+)', html):
391395
if "@" in email:
392396
domain = email.split("@")[1].lower().rstrip("\\/.")
393397
if domain not in SKIP_DOMAINS:
394398
domains.add(domain)
395399

400+
# typo3 obfuscated emails
396401
for encoded in TYPO3_RE.findall(html):
397402
for offset in range(-25, 26):
398403
decoded = decrypt_typo3(encoded, offset)
@@ -403,8 +408,11 @@ def extract_email_domains(html: str) -> set[str]:
403408
domains.add(domain)
404409
break
405410

406-
for match in re.findall(r'[\w.-]+\s*[\[(]at[\])]\s*[\w.-]+\.\w+', html, re.IGNORECASE):
407-
normalized = re.sub(r'\s*[\[(]at[\])]\s*', '@', match, flags=re.IGNORECASE)
411+
# user(at)domain.ch and user[at]domain.ch variants
412+
for match in re.findall(
413+
r"[\w.-]+\s*[\[(]at[\])]\s*[\w.-]+\.\w+", html, re.IGNORECASE
414+
):
415+
normalized = re.sub(r"\s*[\[(]at[\])]\s*", "@", match, flags=re.IGNORECASE)
408416
if "@" in normalized:
409417
domain = normalized.split("@")[1].lower()
410418
if domain not in SKIP_DOMAINS:
@@ -433,6 +441,51 @@ def build_urls(domain: str) -> list[str]:
433441
return urls
434442

435443

444+
def _is_ssl_error(exc: BaseException) -> bool:
445+
"""Check if an exception (or any in its chain) is an SSL verification error."""
446+
current: BaseException | None = exc
447+
while current is not None:
448+
if isinstance(current, ssl.SSLCertVerificationError):
449+
return True
450+
# Some builds wrap the error as a string only
451+
if "CERTIFICATE_VERIFY_FAILED" in str(current):
452+
return True
453+
current = current.__cause__ if current.__cause__ is not current else None
454+
return False
455+
456+
457+
async def _fetch_insecure(url: str) -> httpx.Response:
458+
"""Fetch a URL with SSL verification disabled (single request)."""
459+
with warnings.catch_warnings():
460+
warnings.filterwarnings("ignore", message="Unverified HTTPS request")
461+
async with httpx.AsyncClient(verify=False) as insecure_client:
462+
return await insecure_client.get(url, follow_redirects=True, timeout=15)
463+
464+
465+
def _process_scrape_response(
466+
r: httpx.Response,
467+
domain: str,
468+
all_domains: set[str],
469+
redirect_domain: str | None,
470+
) -> tuple[set[str], str | None]:
471+
"""Extract emails and detect redirects from a scrape response.
472+
473+
Mutates all_domains in place. Returns updated (all_domains, redirect_domain).
474+
"""
475+
if r.status_code != 200:
476+
return all_domains, redirect_domain
477+
478+
if redirect_domain is None:
479+
final_domain = url_to_domain(str(r.url))
480+
if final_domain and final_domain != domain:
481+
redirect_domain = final_domain
482+
logger.info("Redirect detected: {} -> {}", domain, redirect_domain)
483+
484+
domains = extract_email_domains(r.text)
485+
all_domains |= domains
486+
return all_domains, redirect_domain
487+
488+
436489
async def scrape_email_domains(
437490
client: httpx.AsyncClient, domain: str
438491
) -> tuple[set[str], str | None]:
@@ -453,24 +506,27 @@ async def scrape_email_domains(
453506
for url in urls:
454507
try:
455508
r = await client.get(url, follow_redirects=True, timeout=15)
456-
if r.status_code != 200:
509+
except httpx.ConnectError as exc:
510+
if _is_ssl_error(exc):
511+
logger.info("SSL error on {}, retrying without verification", url)
512+
try:
513+
r = await _fetch_insecure(url)
514+
except Exception as retry_exc:
515+
logger.debug("Insecure retry {} failed: {}", url, retry_exc)
516+
continue
517+
else:
518+
logger.debug("Scrape {} failed: {}", url, exc)
457519
continue
458-
459-
# Detect cross-domain redirect (only from first successful response)
460-
if redirect_domain is None:
461-
final_domain = url_to_domain(str(r.url))
462-
if final_domain and final_domain != domain:
463-
redirect_domain = final_domain
464-
logger.info("Redirect detected: {} -> {}", domain, redirect_domain)
465-
466-
domains = extract_email_domains(r.text)
467-
all_domains |= domains
468-
if all_domains:
469-
return all_domains, redirect_domain
470520
except Exception as exc:
471521
logger.debug("Scrape {} failed: {}", url, exc)
472522
continue
473523

524+
all_domains, redirect_domain = _process_scrape_response(
525+
r, domain, all_domains, redirect_domain
526+
)
527+
if all_domains:
528+
return all_domains, redirect_domain
529+
474530
return all_domains, redirect_domain
475531

476532

‎tests/test_resolve.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import stamina
88

99
from mail_sovereignty.resolve import (
10+
_is_ssl_error,
11+
_process_scrape_response,
1012
build_urls,
1113
decrypt_typo3,
1214
detect_website_mismatch,
@@ -1075,3 +1077,150 @@ async def test_logs_bfs_only_warning(self, tmp_path, caplog):
10751077
"municipalities in BFS but missing from Wikidata" in msg
10761078
for msg in caplog.messages
10771079
)
1080+
1081+
1082+
# ── _process_scrape_response() ────────────────────────────────────────
1083+
1084+
1085+
class TestProcessScrapeResponse:
1086+
def test_non_200_returns_unchanged(self):
1087+
r = httpx.Response(404, request=httpx.Request("GET", "https://example.ch"))
1088+
domains, redirect = _process_scrape_response(r, "example.ch", set(), None)
1089+
assert domains == set()
1090+
assert redirect is None
1091+
1092+
def test_200_extracts_email_and_redirect(self):
1093+
r = httpx.Response(
1094+
200,
1095+
text="Contact: info@3908.ch",
1096+
request=httpx.Request("GET", "https://www.3908.ch/"),
1097+
)
1098+
domains, redirect = _process_scrape_response(
1099+
r, "gemeinde-saas-balen.ch", set(), None
1100+
)
1101+
assert "3908.ch" in domains
1102+
assert redirect == "3908.ch"
1103+
1104+
def test_200_same_domain_no_redirect(self):
1105+
r = httpx.Response(
1106+
200,
1107+
text="Contact: info@mygemeinde.ch",
1108+
request=httpx.Request("GET", "https://www.mygemeinde.ch/"),
1109+
)
1110+
domains, redirect = _process_scrape_response(r, "mygemeinde.ch", set(), None)
1111+
assert "mygemeinde.ch" in domains
1112+
assert redirect is None
1113+
1114+
def test_preserves_existing_redirect(self):
1115+
r = httpx.Response(
1116+
200,
1117+
text="Contact: info@other.ch",
1118+
request=httpx.Request("GET", "https://www.other.ch/"),
1119+
)
1120+
domains, redirect = _process_scrape_response(
1121+
r, "example.ch", set(), "already.ch"
1122+
)
1123+
assert "other.ch" in domains
1124+
assert redirect == "already.ch"
1125+
1126+
1127+
# ── _is_ssl_error() ─────────────────────────────────────────────────
1128+
1129+
1130+
class TestIsSslError:
1131+
def test_direct_ssl_error(self):
1132+
import ssl
1133+
1134+
exc = ssl.SSLCertVerificationError("certificate verify failed")
1135+
assert _is_ssl_error(exc) is True
1136+
1137+
def test_nested_ssl_error(self):
1138+
import ssl
1139+
1140+
ssl_exc = ssl.SSLCertVerificationError("certificate verify failed")
1141+
connect_exc = httpx.ConnectError("SSL error")
1142+
connect_exc.__cause__ = ssl_exc
1143+
assert _is_ssl_error(connect_exc) is True
1144+
1145+
def test_non_ssl_error(self):
1146+
exc = ConnectionRefusedError("Connection refused")
1147+
assert _is_ssl_error(exc) is False
1148+
1149+
def test_string_fallback(self):
1150+
exc = Exception("CERTIFICATE_VERIFY_FAILED in handshake")
1151+
assert _is_ssl_error(exc) is True
1152+
1153+
1154+
# ── SSL retry in scrape_email_domains() ──────────────────────────────
1155+
1156+
1157+
class TestSslRetry:
1158+
@pytest.mark.asyncio
1159+
async def test_ssl_error_triggers_insecure_retry(self):
1160+
"""SSL error should trigger an insecure retry that recovers."""
1161+
import ssl
1162+
1163+
ssl_exc = ssl.SSLCertVerificationError("certificate verify failed")
1164+
connect_exc = httpx.ConnectError("SSL handshake failed")
1165+
connect_exc.__cause__ = ssl_exc
1166+
1167+
client = AsyncMock()
1168+
client.get = AsyncMock(side_effect=connect_exc)
1169+
1170+
fake_response = AsyncMock()
1171+
fake_response.status_code = 200
1172+
fake_response.text = "Contact: gemeinde@3908.ch"
1173+
fake_response.url = httpx.URL("https://www.3908.ch/")
1174+
1175+
with patch(
1176+
"mail_sovereignty.resolve._fetch_insecure",
1177+
new_callable=AsyncMock,
1178+
return_value=fake_response,
1179+
) as mock_fetch:
1180+
domains, redirect = await scrape_email_domains(
1181+
client, "gemeinde-saas-balen.ch"
1182+
)
1183+
1184+
assert "3908.ch" in domains
1185+
assert redirect == "3908.ch"
1186+
mock_fetch.assert_called()
1187+
1188+
@pytest.mark.asyncio
1189+
async def test_non_ssl_connect_error_no_retry(self):
1190+
"""Non-SSL ConnectError should not trigger insecure retry."""
1191+
connect_exc = httpx.ConnectError("Connection refused")
1192+
1193+
client = AsyncMock()
1194+
client.get = AsyncMock(side_effect=connect_exc)
1195+
1196+
with patch(
1197+
"mail_sovereignty.resolve._fetch_insecure",
1198+
new_callable=AsyncMock,
1199+
) as mock_fetch:
1200+
domains, redirect = await scrape_email_domains(client, "example.ch")
1201+
1202+
assert domains == set()
1203+
assert redirect is None
1204+
mock_fetch.assert_not_called()
1205+
1206+
@pytest.mark.asyncio
1207+
async def test_ssl_retry_failure_continues(self):
1208+
"""If insecure retry also fails, scrape should continue gracefully."""
1209+
import ssl
1210+
1211+
ssl_exc = ssl.SSLCertVerificationError("certificate verify failed")
1212+
connect_exc = httpx.ConnectError("SSL handshake failed")
1213+
connect_exc.__cause__ = ssl_exc
1214+
1215+
client = AsyncMock()
1216+
client.get = AsyncMock(side_effect=connect_exc)
1217+
1218+
with patch(
1219+
"mail_sovereignty.resolve._fetch_insecure",
1220+
new_callable=AsyncMock,
1221+
side_effect=httpx.ConnectError("still broken"),
1222+
):
1223+
domains, redirect = await scrape_email_domains(client, "example.ch")
1224+
1225+
assert domains == set()
1226+
assert redirect is None

0 commit comments

Comments
 (0)