Skip to content

Commit 58280eb

Browse files
MilesMorelclaude
andcommitted
Retry RescueGroups API requests with backoff
A transient ReadTimeout from api.rescuegroups.org crashed the entire run (it's the only non-debug source), since main.run() only catches ValueError. Mount an HTTPAdapter with urllib3 Retry on the request session so transient timeouts and 5xx/429 responses are retried with exponential backoff before giving up. POST is opted into allowed_methods (not retried by default), and raise_on_status=False preserves the existing raise_for_status() error path. Fixes #116 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent efb4ca1 commit 58280eb

1 file changed

Lines changed: 25 additions & 1 deletion

File tree

adoption_sources/rescue_groups.py

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
from typing import Iterator
1212

1313
import requests
14+
from requests.adapters import HTTPAdapter
15+
from urllib3.util.retry import Retry
1416

1517
from abstractions import AdoptablePet, PetSource
1618
from config import CITY_NAME, CITY_STATE, POSTAL_CODE
@@ -21,6 +23,27 @@
2123
# website; those should never be posted. Add new names here as we encounter them.
2224
PLACEHOLDER_NAMES: tuple[str, ...] = ("more dogs soon!",)
2325

26+
# The RescueGroups API occasionally times out or returns a transient 5xx. A
27+
# single hiccup shouldn't fail the whole run, so retry a few times with
28+
# exponential backoff (0s, 2s, 4s, 8s between attempts).
29+
RETRY_TOTAL = 4
30+
RETRY_BACKOFF_FACTOR = 1
31+
32+
33+
def _session_with_retries() -> requests.Session:
34+
"""Build a requests Session that retries transient errors with backoff."""
35+
retry = Retry(
36+
total=RETRY_TOTAL,
37+
backoff_factor=RETRY_BACKOFF_FACTOR,
38+
status_forcelist=(429, 500, 502, 503, 504),
39+
# We only POST, so POST must be opted in (it isn't retried by default).
40+
allowed_methods=frozenset({"POST"}),
41+
raise_on_status=False,
42+
)
43+
session = requests.Session()
44+
session.mount("https://", HTTPAdapter(max_retries=retry))
45+
return session
46+
2447

2548
class SourceRescueGroups(PetSource):
2649
"""
@@ -92,7 +115,8 @@ def fetch_pets(self) -> Iterator[AdoptablePet]:
92115
f"Fetching {self.species} from RescueGroups within {self.radius_miles} miles of {self.postal_code}"
93116
)
94117

95-
response = requests.post(url, json=payload, headers=headers, timeout=30)
118+
session = _session_with_retries()
119+
response = session.post(url, json=payload, headers=headers, timeout=30)
96120
response.raise_for_status()
97121

98122
body = response.json()

0 commit comments

Comments
 (0)