feat: include custom user agent in requests (#50)

This makes it possible to identify traffic from us specifically, which
can be useful for api providers with gauging our impact on their
infrastructure and also (hopefully) make us seem more like a legitimate
and positive consumer rather than just "yet another
`python-requests/<version>` scraper"

Author: G-Rath

ruff.toml

@@ -18,5 +18,6 @@ select = [
 [lint.isort]
 known-first-party = [
   'generate_osv_advisories',
+  'user_agent',
   'typings'
 ]

scripts/download_sa_advisories.py

@@ -14,6 +14,7 @@
 import requests
 
 from typings import drupal
+from user_agent import user_agent
 
 
 def get_most_recent_changed_timestamp() -> int:
@@ -51,7 +52,7 @@ def download_sa_advisories_from_rest_api(last_modified_timestamp: int) -> None:
   fetch_again = True
   while fetch_again:
     print(f'fetching {url}')
-    response = requests.get(url)
+    response = requests.get(url, headers={'user-agent': user_agent})
     if response.status_code == 200:
       data: drupal.ApiResponse[drupal.Advisory] = response.json()
       for item in data['list']:

scripts/generate_osv_advisories.py

@@ -17,6 +17,7 @@
 from markdownify import markdownify
 
 from typings import drupal, osv
+from user_agent import user_agent
 
 
 def fetch_drupal_node(nid: str) -> drupal.Node:
@@ -31,7 +32,10 @@ def fetch_drupal_node(nid: str) -> drupal.Node:
   except FileNotFoundError as e:
     os.makedirs('cache/nodes', exist_ok=True)
     print(f' *- fetching https://www.drupal.org/api-d7/node/{nid}.json')
-    resp = requests.get(f'https://www.drupal.org/api-d7/node/{nid}.json')
+    resp = requests.get(
+      f'https://www.drupal.org/api-d7/node/{nid}.json',
+      headers={'user-agent': user_agent},
+    )
 
     if resp.status_code == 200:
       node: drupal.Node = resp.json()

scripts/precache_nodes.py

@@ -13,6 +13,7 @@
 import requests
 
 from typings import drupal
+from user_agent import user_agent
 
 
 def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]:
@@ -24,7 +25,7 @@ def fetch_drupal_nodes(nids: list[str]) -> list[drupal.Node]:
   for nid in nids:
     url += f'nid[]={nid}&'
 
-  resp = requests.get(url)
+  resp = requests.get(url, headers={'user-agent': user_agent})
 
   if resp.status_code == 200:
     items = typing.cast(drupal.ApiResponse[drupal.Node], resp.json())['list']

scripts/user_agent.py

@@ -0,0 +1,7 @@
+import os
+
+user_agent = 'drupal-advisory-database/'
+if 'CI' in os.environ:
+  user_agent += 'ci'
+else:
+  user_agent += 'local'