fix(httpx): try passing headers to httpx requests to prevent 403 from hetzner (#303)

Author: luabida

pysus/api/ducklake/catalog/adapters.py

@@ -158,7 +158,8 @@ async def _download_catalog(
         force: bool = False,
         callback: Callable[[int, int], None] | None = None,
     ) -> None:
-        url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}"
+        remote = str(remote_path).replace("\\", "/")
+        url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote}"
 
         if local_path.exists() and not force:
             try:
@@ -170,7 +171,17 @@ async def _download_catalog(
 
         remote_size = 0
         if local_size != -1:
-            async with httpx.AsyncClient(follow_redirects=True) as client:
+            headers = {
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/120.0.0.0 Safari/537.36"
+                ),
+                "Referer": "https://github.com/AlertaDengue/PySUS",
+            }
+            async with httpx.AsyncClient(
+                headers=headers, follow_redirects=True
+            ) as client:
                 try:
                     head = await client.head(url)
 
@@ -189,7 +200,7 @@ async def _download_catalog(
 
         try:
             await download_http(
-                remote_path=remote_path,
+                remote_path=remote,
                 local_path=local_path,
                 callback=callback,
             )

pysus/api/ducklake/functional.py

@@ -14,15 +14,25 @@ async def download_http(
     local_path: Path,
     callback: Callable[[int, int], None] | None = None,
 ) -> None:
+    remote_path = str(remote_path).replace("\\", "/")
     url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}"
     max_retries = 5
 
     timeout = httpx.Timeout(15.0, read=60.0, write=20.0, connect=15.0)
     limits = httpx.Limits(max_keepalive_connections=5, max_connections=10)
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Referer": "https://github.com/AlertaDengue/PySUS",
+    }
 
     for attempt in range(max_retries):
         try:
             async with httpx.AsyncClient(
+                headers=headers,
                 follow_redirects=True,
                 verify=False,
                 limits=limits,