Skip to content

refactor: clean up sa downloader script #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
G-Rath merged 4 commits into from
Aug 13, 2025
Merged

refactor: clean up sa downloader script #119

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
746e9e7
refactor: eliminate `else`s
G-Rath Aug 6, 2025
16e1fcf
refactor: reuse variable
G-Rath Aug 6, 2025
a4a7ae4
refactor: eliminate remaining `else`
G-Rath Aug 6, 2025
c19ca5b
refactor: remove unneeded check
G-Rath Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 20 additions & 25 deletions scripts/download_sa_advisories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,34 +49,29 @@ def download_sa_advisories_from_rest_api(last_modified_timestamp: int) -> None:

print(f'fetching sa advisories modified after {last_modified_timestamp}')
url = 'https://www.drupal.org/api-d7/node.json?type=sa&sort=changed&direction=DESC&field_is_psa=0'
fetch_again = True
while fetch_again:
while url != '':
print(f'fetching {url}')
response = requests.get(url, headers={'user-agent': user_agent})
if response.status_code == 200:
data: drupal.ApiResponse[drupal.Advisory] = response.json()
for item in data['list']:
changed = int(item['changed'])
if changed > last_modified_timestamp:
advisory_id = determine_sa_id(item)
print(
f' |- updating cache/advisories/{advisory_id}.json as {item["url"]} has changed'
)
with open(f'cache/advisories/{advisory_id}.json', 'w') as f:
json.dump(item, f)
f.write('\n')
else:
# We have reached the last modified entry.
fetch_again = False
print(' \\- finished processing page')
if 'next' in data and data['next'] != '':
url = data['next'].replace('api-d7/node?', 'api-d7/node.json?')
else:
print('finished processing new and updated advisories')
fetch_again = False
else:
if response.status_code != 200:
print(f'X API responded {response.status_code}')
fetch_again = False
break
data: drupal.ApiResponse[drupal.Advisory] = response.json()
url = data.get('next', '').replace('api-d7/node?', 'api-d7/node.json?')
for item in data['list']:
changed = int(item['changed'])
if changed <= last_modified_timestamp:
# We have reached the last modified entry.
url = ''
break
advisory_id = determine_sa_id(item)
print(
f' |- updating cache/advisories/{advisory_id}.json as {item["url"]} has changed'
)
with open(f'cache/advisories/{advisory_id}.json', 'w') as f:
json.dump(item, f)
f.write('\n')
print(' \\- finished processing page')
print('finished processing new and updated advisories')


most_recent_changed_time = get_most_recent_changed_timestamp()
Expand Down