Reduce 429 error responses when checking links

- set concurrency to 1
- retry 429 responses with retry-after header
- set bearer token for GitHub links

Author: philippthun

.github/workflows/docs_test.yml

@@ -34,6 +34,8 @@ jobs:
         working-directory: docs/v2
         run: ./render.sh
       - name: Run docs tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           bundle config set --local without 'development'
           bundle install

docs/v3/gulpfile.js

@@ -4,8 +4,8 @@ import gulp from 'gulp';
 import express from 'express';
 import { glob } from 'glob';
 import { LinkChecker } from 'linkinator';
-
 import * as cheerio from 'cheerio';
+import { instance as gaxios } from 'gaxios';
 
 function displayErrors(err, stdout, stderr) {
   if (err) {
@@ -85,15 +85,33 @@ async function checkPathAndExit(path, options, done) {
 
   const url = 'http://localhost:8001/';
 
+  const githubToken = process.env.GITHUB_TOKEN;
+  console.log('GITHUB_TOKEN is', githubToken ? 'SET' : 'NOT SET');
+
   const config = {
     path: url,
     linksToSkip: options.linksToSkip || [],
     recurse: options.recurse,
     silent: options.silent,
     markdown: options.markdown,
+    concurrency: 1,
+    retry: true,
   };
 
   try {
+    gaxios.interceptors.request.add({
+      resolved: (config) => {
+        if (githubToken && config.url?.includes('github.com')) {
+          config.headers = {
+            ...(config.headers || {}),
+            authorization: `Bearer ${githubToken}`,
+          };
+        }
+        return config;
+      },
+      rejected: (error) => Promise.reject(error),
+    });
+
     const checker = new LinkChecker();
 
     if (path === '../v2') {