refactor(ng-dev): Add github api retries for increased reliability

Github has been increasingly flakey lately resulting in partial merges and all sorts of issues with failures in the tooling. This adds a retry system to ensure github API calls retry up to 3 times if they fail.

Author: thePunderWoman

.github/local-actions/branch-manager/main.js

@@ -11,6 +11,7 @@ import type {OctokitOptions} from '@octokit/core';
 import {Octokit} from '@octokit/rest';
 import {RequestParameters} from '@octokit/types';
 import {RequestError} from '@octokit/request-error';
+import {GraphqlResponseError} from '@octokit/graphql';
 import {query} from 'typed-graphqlify';
 import {Log} from '../logging';
 
@@ -28,6 +29,60 @@ export interface GithubRepo {
   name: string;
 }
 
+/** Helper to invoke an async function with retries. */
+async function invokeWithRetry<T>(fn: () => Promise<T>, retries = 3, delay = 1000): Promise<T> {
+  let attempt = 0;
+  while (attempt < retries) {
+    try {
+      return await fn();
+    } catch (e) {
+      attempt++;
+      if (attempt >= retries) {
+        throw e;
+      }
+
+      // Do not retry valid 4xx Client Errors (especially 404 Not Found)
+      if (isGithubApiError(e) && e.status < 500) {
+        throw e;
+      }
+
+      // Do not retry permanent GraphQL errors
+      if (
+        e instanceof GraphqlResponseError &&
+        e.errors?.every((err) =>
+          ['NOT_FOUND', 'FORBIDDEN', 'BAD_USER_INPUT', 'UNAUTHENTICATED'].includes(err.type!),
+        )
+      ) {
+        throw e;
+      }
+
+      Log.warn(`GitHub API call failed (attempt ${attempt}/${retries}). Retrying in ${delay}ms...`);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+  throw new Error('Unreachable');
+}
+
+/** Creates a proxy that intercepts function calls and applies retries. */
+function createRetryProxy<T extends object>(target: T): T {
+  return new Proxy(target, {
+    get(targetObj, prop, receiver) {
+      const value = Reflect.get(targetObj, prop, receiver);
+      if (typeof value === 'function') {
+        return new Proxy(value, {
+          apply(targetFn, thisArg, argArray) {
+            return invokeWithRetry(() => (targetFn as Function).apply(targetObj, argArray));
+          },
+        });
+      }
+      if (typeof value === 'object' && value !== null) {
+        return createRetryProxy(value);
+      }
+      return value;
+    },
+  });
+}
+
 /** A Github client for interacting with the Github APIs. */
 export class GithubClient {
   /** The octokit instance actually performing API requests. */
@@ -43,18 +98,18 @@ export class GithubClient {
     ...this._octokitOptions,
   });
 
-  readonly pulls: Octokit['pulls'] = this._octokit.pulls;
-  readonly orgs: Octokit['orgs'] = this._octokit.orgs;
-  readonly repos: Octokit['repos'] = this._octokit.repos;
-  readonly issues: Octokit['issues'] = this._octokit.issues;
-  readonly git: Octokit['git'] = this._octokit.git;
-  readonly rateLimit: Octokit['rateLimit'] = this._octokit.rateLimit;
-  readonly teams: Octokit['teams'] = this._octokit.teams;
-  readonly search: Octokit['search'] = this._octokit.search;
-  readonly rest: Octokit['rest'] = this._octokit.rest;
+  readonly pulls: Octokit['pulls'] = createRetryProxy(this._octokit.pulls);
+  readonly orgs: Octokit['orgs'] = createRetryProxy(this._octokit.orgs);
+  readonly repos: Octokit['repos'] = createRetryProxy(this._octokit.repos);
+  readonly issues: Octokit['issues'] = createRetryProxy(this._octokit.issues);
+  readonly git: Octokit['git'] = createRetryProxy(this._octokit.git);
+  readonly rateLimit: Octokit['rateLimit'] = createRetryProxy(this._octokit.rateLimit);
+  readonly teams: Octokit['teams'] = createRetryProxy(this._octokit.teams);
+  readonly search: Octokit['search'] = createRetryProxy(this._octokit.search);
+  readonly rest: Octokit['rest'] = createRetryProxy(this._octokit.rest);
   readonly paginate: Octokit['paginate'] = this._octokit.paginate;
-  readonly checks: Octokit['checks'] = this._octokit.checks;
-  readonly users: Octokit['users'] = this._octokit.users;
+  readonly checks: Octokit['checks'] = createRetryProxy(this._octokit.checks);
+  readonly users: Octokit['users'] = createRetryProxy(this._octokit.users);
 
   constructor(private _octokitOptions?: OctokitOptions) {}
 }
@@ -76,7 +131,9 @@ export class AuthenticatedGithubClient extends GithubClient {
 
   /** Perform a query using Github's Graphql API. */
   async graphql<T extends GraphqlQueryObject>(queryObject: T, params: RequestParameters = {}) {
-    return (await this._graphql(query(queryObject).toString(), params)) as T;
+    return invokeWithRetry(async () => {
+      return (await this._graphql(query(queryObject).toString(), params)) as T;
+    });
   }
 }