Skip to content

Commit fdd4011

Browse files
namedgraphclaude
andcommitted
Add RetryAfterHandler to respect 429 Retry-After responses
Plugs into the urllib opener chain (same pattern as HTTPRedirectHandler308) so both LinkedDataClient and SPARQLClient automatically wait and retry on HTTP 429 responses, honouring integer-seconds and HTTP-date Retry-After header values. Also fixes missing urllib.parse import in SPARQLClient. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent c17aee9 commit fdd4011

1 file changed

Lines changed: 30 additions & 1 deletion

File tree

src/web_algebra/client.py

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
from typing import Optional
22
import ssl
33
import json
4+
import time
5+
import urllib.error
6+
import urllib.parse
47
import urllib.request
8+
from datetime import datetime, timezone
9+
from email.utils import parsedate_to_datetime
510
from http.client import HTTPResponse
611
from rdflib import Graph
712
from rdflib.plugins.sparql.parser import parseQuery
@@ -25,6 +30,28 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
2530
return super().redirect_request(req, fp, code, msg, headers, newurl)
2631

2732

33+
class RetryAfterHandler(urllib.request.BaseHandler):
34+
def __init__(self, max_retries: int = 3):
35+
self.max_retries = max_retries
36+
self._retry_counts: dict = {}
37+
38+
def http_error_429(self, req, fp, code, msg, hdrs):
39+
key = req.full_url
40+
count = self._retry_counts.get(key, 0)
41+
if count >= self.max_retries:
42+
self._retry_counts.pop(key, None)
43+
raise urllib.error.HTTPError(req.full_url, code, msg, hdrs, fp)
44+
self._retry_counts[key] = count + 1
45+
retry_after = hdrs.get("Retry-After", "1")
46+
try:
47+
delay = float(retry_after)
48+
except ValueError:
49+
retry_dt = parsedate_to_datetime(retry_after)
50+
delay = max(0.0, (retry_dt - datetime.now(tz=timezone.utc)).total_seconds())
51+
time.sleep(delay)
52+
return self.parent.open(req)
53+
54+
2855
class LinkedDataClient:
2956
def __init__(
3057
self,
@@ -57,6 +84,7 @@ def __init__(
5784
self.opener = urllib.request.build_opener(
5885
urllib.request.HTTPSHandler(context=self.ssl_context),
5986
HTTPRedirectHandler308(),
87+
RetryAfterHandler(),
6088
)
6189

6290
# Add proper User-Agent header for external services like Wikidata
@@ -195,7 +223,8 @@ def __init__(
195223
self.ssl_context.verify_mode = ssl.CERT_NONE
196224

197225
self.opener = urllib.request.build_opener(
198-
urllib.request.HTTPSHandler(context=self.ssl_context)
226+
urllib.request.HTTPSHandler(context=self.ssl_context),
227+
RetryAfterHandler(),
199228
)
200229

201230
# Add proper User-Agent header for external services like Wikidata

0 commit comments

Comments
 (0)