feat: retry request if api returns a 429 when precaching nodes

Author: G-Rath

scripts/precache_nodes.py

@@ -7,6 +7,7 @@
 
 import json
 import os
+import time
 import typing
 from itertools import batched
 
@@ -16,7 +17,7 @@
 from user_agent import user_agent
 
 
-def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]:
+def fetch_drupal_nodes(nids: list[str], retry=True) -> list[drupal.Node]:
   """
   Fetches a node from drupal.org by its id
   """
@@ -27,6 +28,13 @@ def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]:
 
   resp = requests.get(url, headers={'user-agent': user_agent})
 
+  if retry and resp.status_code == 429:
+    seconds = int(resp.headers.get('Retry-After', 0))
+    print(f'  |* (waiting {seconds} seconds before retrying)')
+    time.sleep(seconds)
+
+    return fetch_drupal_nodes(nids, retry=False)
+
   if resp.status_code == 200:
     items = typing.cast(drupal.ApiResponse[drupal.Node], resp.json())['list']