feat: retry request if api returns a 429 when precaching nodes (#90)

We've had a couple of our (re)generation runs fail when precaching the
Drupal nodes due to 429s - we might as well include a basic "retry once"
attempt to help avoid this. While it's possible this could happen for
any API request it should be very unlikely except for this one which is
why I've not changed them.

In the long-run if this becomes more regular (or requested by the Drupal
team) we'll probably move to a more dedicated HTTP client implementation
to handle this across the board

Author: G-Rath

scripts/precache_nodes.py

@@ -7,6 +7,7 @@
 
 import json
 import os
+import time
 import typing
 from itertools import batched
 
@@ -16,7 +17,7 @@
 from user_agent import user_agent
 
 
-def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]:
+def fetch_drupal_nodes(nids: list[str], retry: bool = True) -> list[drupal.Node]:
   """
   Fetches a node from drupal.org by its id
   """
@@ -27,6 +28,13 @@ def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]:
 
   resp = requests.get(url, headers={'user-agent': user_agent})
 
+  if retry and resp.status_code == 429:
+    seconds = int(resp.headers.get('Retry-After', 0))
+    print(f' |* (waiting {seconds} seconds before retrying)')
+    time.sleep(seconds)
+
+    return fetch_drupal_nodes(nids, retry=False)
+
   if resp.status_code == 200:
     items = typing.cast(drupal.ApiResponse[drupal.Node], resp.json())['list']