From 6d55485328e4cc676e7f94f23e997e00e541b8d1 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 24 Jun 2025 08:47:10 +1200 Subject: [PATCH] feat: retry request if api returns a 429 when precaching nodes --- scripts/precache_nodes.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/precache_nodes.py b/scripts/precache_nodes.py index 8c8d143a..b7dd23da 100755 --- a/scripts/precache_nodes.py +++ b/scripts/precache_nodes.py @@ -7,6 +7,7 @@ import json import os +import time import typing from itertools import batched @@ -16,7 +17,7 @@ from user_agent import user_agent -def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]: +def fetch_drupal_nodes(nids: list[str], retry: bool = True) -> list[drupal.Node]: """ Fetches a node from drupal.org by its id """ @@ -27,6 +28,13 @@ def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]: resp = requests.get(url, headers={'user-agent': user_agent}) + if retry and resp.status_code == 429: + seconds = int(resp.headers.get('Retry-After', 0)) + print(f' |* (waiting {seconds} seconds before retrying)') + time.sleep(seconds) + + return fetch_drupal_nodes(nids, retry=False) + if resp.status_code == 200: items = typing.cast(drupal.ApiResponse[drupal.Node], resp.json())['list']