Merge single RescueGroups API call into add-cats-to-display.

Smoss · cursoragent · Smoss · commit 8ad7ba65a533 · 2026-06-09T20:29:54.000-04:00
Use one filterProcessing request for dogs and cats while keeping top-level imports and debug fixtures.

Co-authored-by: Cursor &lt;cursoragent@cursor.com&gt;
diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py
@@ -8,12 +8,15 @@
 import logging
 import os
 import re
+from collections.abc import Sequence
 from typing import Iterator
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from abstractions import AdoptablePet, PetSource
-from config import CITY_NAME, CITY_STATE, POSTAL_CODE
+from config import CITY_NAME, CITY_STATE, PET_SPECIES, POSTAL_CODE, RESCUEGROUPS_LIMIT
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +26,39 @@
 
 SPECIES_SINGULAR = {"dogs": "dog", "cats": "cat"}
 
+# The RescueGroups API occasionally times out or returns a transient 5xx. A
+# single hiccup shouldn't fail the whole run, so retry a few times with
+# exponential backoff (0s, 2s, 4s, 8s between attempts).
+RETRY_TOTAL = 4
+RETRY_BACKOFF_FACTOR = 1
+
+
+def _session_with_retries() -> requests.Session:
+    """Build a requests Session that retries transient errors with backoff."""
+    retry = Retry(
+        total=RETRY_TOTAL,
+        backoff_factor=RETRY_BACKOFF_FACTOR,
+        status_forcelist=(429, 500, 502, 503, 504),
+        # We only POST, so POST must be opted in (it isn't retried by default).
+        allowed_methods=frozenset({"POST"}),
+        raise_on_status=False,
+    )
+    session = requests.Session()
+    session.mount("https://", HTTPAdapter(max_retries=retry))
+    return session
+
+
+def _build_species_filters(species: Sequence[str]) -> tuple[list[dict], str]:
+    """Build RescueGroups filters and filterProcessing for an OR species search."""
+    filters = [
+        {"fieldName": "species.plural", "operation": "equal", "criteria": plural}
+        for plural in species
+    ]
+    if not filters:
+        raise ValueError("At least one species is required")
+    filter_processing = " OR ".join(str(index) for index in range(1, len(filters) + 1))
+    return filters, filter_processing
+
 
 class SourceRescueGroups(PetSource):
     """
@@ -38,20 +74,20 @@ def __init__(
         api_key: str | None = None,
         postal_code: str = POSTAL_CODE,
         radius_miles: int = 50,
-        species: str = "dogs",  # "dogs" or "cats"
-        limit: int = 25,
+        species: Sequence[str] | None = None,
+        limit: int = RESCUEGROUPS_LIMIT,
         location_label: str = f"{CITY_NAME}, {CITY_STATE}",
     ):
         self._api_key = api_key or os.environ.get("CUTEPETSBOSTON_RESCUEGROUPS_API_KEY")
         self.postal_code = postal_code
         self.radius_miles = radius_miles
-        self.species = species
+        self.species = tuple(species if species is not None else PET_SPECIES)
         self.limit = limit
         self.location_label = location_label
 
     @property
     def source_name(self) -> str:
-        return f"RescueGroups ({self.species})"
+        return f"RescueGroups ({', '.join(self.species)})"
 
     def fetch_pets(self) -> Iterator[AdoptablePet]:
         """
@@ -69,71 +105,97 @@ def fetch_pets(self) -> Iterator[AdoptablePet]:
                 "RescueGroups API key not configured. "
                 "Set CUTEPETSBOSTON_RESCUEGROUPS_API_KEY environment variable."
             )
-        
+
         url = (
-            f"{self.BASE_URL}/available/{self.species}/haspic"
-            f"?include=orgs,breeds,locations"
+            f"{self.BASE_URL}/available/haspic"
+            f"?include=orgs,breeds,locations,species"
             f"&sort=random"
             f"&limit={self.limit}"
         )
         headers = {
             "Content-Type": "application/vnd.api+json",
             "Authorization": self._api_key,
         }
+        species_filters, filter_processing = _build_species_filters(self.species)
         payload = {
             "data": {
                 "filterRadius": {
                     "miles": self.radius_miles,
                     "postalcode": self.postal_code,
-                }
+                },
+                "filters": species_filters,
+                "filterProcessing": filter_processing,
             }
         }
 
-
         logger.info(
-            f"Fetching {self.species} from RescueGroups within {self.radius_miles} miles of {self.postal_code}"
+            "Fetching %s from RescueGroups within %s miles of %s",
+            ", ".join(self.species),
+            self.radius_miles,
+            self.postal_code,
         )
 
-        response = requests.post(url, json=payload, headers=headers, timeout=30)
+        session = _session_with_retries()
+        response = session.post(url, json=payload, headers=headers, timeout=30)
         response.raise_for_status()
 
         body = response.json()
         data = body.get("data", [])
-        logger.info(f"Received {len(data)} pets from RescueGroups")
+        logger.info("Received %s pets from RescueGroups", len(data))
 
         orgs_by_id = {
             item["id"]: item.get("attributes", {})
             for item in body.get("included", [])
             if item.get("type") == "orgs"
         }
+        species_by_id = {
+            item["id"]: item.get("attributes", {})
+            for item in body.get("included", [])
+            if item.get("type") == "species"
+        }
 
         for animal in data:
-            pet = self._parse_animal(animal, orgs_by_id)
+            pet = self._parse_animal(animal, orgs_by_id, species_by_id)
             if not pet:
                 continue
             if self._is_placeholder_name(pet.name):
-                logger.info(f"Skipping placeholder record: {pet.name!r}")
+                logger.info("Skipping placeholder record: %r", pet.name)
                 continue
             yield pet
 
-    def _parse_animal(self, animal: dict, orgs_by_id: dict) -> AdoptablePet | None:
+    def _parse_animal(
+        self,
+        animal: dict,
+        orgs_by_id: dict,
+        species_by_id: dict,
+    ) -> AdoptablePet | None:
         """Parse a single animal record from the API response."""
         try:
             attrs = animal.get("attributes", {})
             animal_id = animal.get("id", "")
 
-            # Extract and clean the name
             name = self._clean_name(attrs.get("name", "Unknown"))
 
-            species = SPECIES_SINGULAR[self.species]
+            species_id = (
+                animal.get("relationships", {})
+                .get("species", {})
+                .get("data", [{}])[0]
+                .get("id")
+            )
+            if not species_id:
+                logger.warning("Skipping animal %s with no species relationship", animal_id)
+                return None
+
+            plural = species_by_id.get(species_id, {}).get("plural")
+            if plural not in self.species:
+                logger.info("Skipping animal %s with unconfigured species: %r", animal_id, plural)
+                return None
 
-            # Get breed info
-            breed = attrs.get("breedString", attrs.get("breedPrimary", "Mixed"))
+            species = SPECIES_SINGULAR[plural]
 
-            # Clean up description (use text version, not HTML)
+            breed = attrs.get("breedString", attrs.get("breedPrimary", "Mixed"))
             description = self._clean_description(attrs.get("descriptionText", ""))
 
-            # Get adoption_url
             org_id = (
                 animal.get("relationships", {})
                 .get("orgs", {})
@@ -147,13 +209,9 @@ def _parse_animal(self, animal: dict, orgs_by_id: dict) -> AdoptablePet | None:
                 None
             )
 
-            # Get best available image
             image_url = self._get_image_url(attrs)
-
-            # Location of the adoption org
             location = f"{org_attrs.get('city')}, {org_attrs.get('state')}"
 
-
             return AdoptablePet(
                 name=name,
                 species=species,
@@ -168,7 +226,7 @@ def _parse_animal(self, animal: dict, orgs_by_id: dict) -> AdoptablePet | None:
                 pet_id=animal_id,
             )
         except Exception as e:
-            logger.warning(f"Failed to parse animal {animal.get('id', 'unknown')}: {e}")
+            logger.warning("Failed to parse animal %s: %s", animal.get("id", "unknown"), e)
             return None
 
     def _is_placeholder_name(self, name: str) -> bool:
@@ -182,8 +240,6 @@ def _clean_name(self, name: str) -> str:
             "Doli ***Home for the Holidays 1/2 price!" -> "Doli"
             "Kathy" -> "Kathy"
         """
-        # Remove common promotional suffixes
-        # Split on common delimiters and take the first part
         cleaned = re.split(r"\s*[\*\-\|]+\s*", name)[0]
         return cleaned.strip()
 
@@ -192,19 +248,13 @@ def _clean_description(self, description: str) -> str:
         if not description:
             return ""
 
-        # Decode HTML entities
         text = html.unescape(description)
-
-        # Remove &nbsp; and normalize whitespace
         text = text.replace("&nbsp;", " ")
         text = re.sub(r"\s+", " ", text)
-
-        # Remove promotional headers
         text = re.sub(
             r"\*\*Home for the Holidays.*?\*\*", "", text, flags=re.IGNORECASE
         )
 
-        # Trim to reasonable length for social posts
         text = text.strip()
         if len(text) > 500:
             text = text[:497] + "..."
@@ -215,6 +265,5 @@ def _get_image_url(self, attrs: dict) -> str | None:
         """Get the best available image URL."""
         thumbnail = attrs.get("pictureThumbnailUrl")
         if thumbnail:
-            # Request a larger image instead of the 100px thumbnail
             return re.sub(r"\?width=\d+", "?width=800", thumbnail)
         return None
diff --git a/config.py b/config.py
@@ -5,3 +5,6 @@
 
 # RescueGroups API plural endpoint names for species to fetch.
 PET_SPECIES = ("dogs", "cats")
+
+# Single-call limit; roughly matches two per-species calls at 25 each.
+RESCUEGROUPS_LIMIT = 50
diff --git a/main.py b/main.py
@@ -10,7 +10,6 @@
 import requests
 
 from adoption_sources import SourceManual, SourceRescueGroups
-from config import PET_SPECIES
 from social_posters.bluesky import PosterBluesky
 from social_posters.debug import PosterDebug
 from social_posters.instagram import PosterInstagram
@@ -59,9 +58,7 @@ def create_sources(debug=False):
         ]
 
     sources = []
-    for species in PET_SPECIES:
-        sources.append(SourceRescueGroups(species=species))
-
+    sources.append(SourceRescueGroups())
     return sources
 
 
diff --git a/tests/test_main.py b/tests/test_main.py
@@ -90,14 +90,12 @@ def test_run_with_mixed_species_pool(self):
 
 
 class CreateSourcesTests(unittest.TestCase):
-    def test_prod_returns_rescuegroups_for_each_species(self):
+    def test_prod_returns_single_rescuegroups_source(self):
         sources = create_sources(debug=False)
 
-        self.assertEqual(len(sources), 2)
+        self.assertEqual(len(sources), 1)
         self.assertIsInstance(sources[0], SourceRescueGroups)
-        self.assertIsInstance(sources[1], SourceRescueGroups)
-        self.assertEqual(sources[0].species, "dogs")
-        self.assertEqual(sources[1].species, "cats")
+        self.assertEqual(sources[0].species, ("dogs", "cats"))
 
     def test_debug_returns_manual_sources_for_dogs_and_cats(self):
         sources = create_sources(debug=True)
@@ -117,6 +115,5 @@ def test_debug_returns_debug_poster(self):
         self.assertEqual(posters[0].platform_name, "Debug")
 
 
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py
