chore: handle gitlab rate limit errors better

themarolt · themarolt · commit 62d2c786a13e · 2025-10-16T09:44:44.000+02:00
Signed-off-by: Uroš Marolt &lt;uros@marolt.me&gt;
diff --git a/services/libs/integrations/src/integrations/gitlab/api/errorHandler.ts b/services/libs/integrations/src/integrations/gitlab/api/errorHandler.ts
@@ -0,0 +1,111 @@
+import { Logger } from '@crowd/logging'
+import { RateLimitError } from '@crowd/types'
+
+const getHeader = (
+  headers: Record<string, string> | undefined,
+  name: string,
+): string | undefined => {
+  if (!headers) return undefined
+
+  if (headers[name]) return headers[name]
+
+  const lowerName = name.toLowerCase()
+  if (headers[lowerName]) return headers[lowerName]
+
+  const headerKey = Object.keys(headers).find((key) => key.toLowerCase() === lowerName)
+  return headerKey ? headers[headerKey] : undefined
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const handleGitlabError = (err: any, endpoint: string, logger: Logger) => {
+  // Check if this is a rate limit error (429)
+  if (err && err.response && err.response.status === 429) {
+    logger.warn('GitLab API rate limit exceeded')
+    let rateLimitResetSeconds = 60 // Default to 60 seconds if no header is present
+
+    // https://docs.gitlab.com/administration/settings/user_and_ip_rate_limits/#use-a-custom-rate-limit-response
+
+    // Retry-After header in seconds
+    const retryAfter = getHeader(err.response.headers, 'retry-after')
+    if (retryAfter) {
+      rateLimitResetSeconds = parseInt(retryAfter, 10)
+    }
+    // RateLimit-Reset header (Unix timestamp)
+    else {
+      const rateLimitReset = getHeader(err.response.headers, 'ratelimit-reset')
+      if (rateLimitReset) {
+        const resetTimestamp = parseInt(rateLimitReset, 10)
+        const now = Math.floor(Date.now() / 1000)
+        rateLimitResetSeconds = Math.max(resetTimestamp - now, 0) + 5 // Add 5 second buffer
+      }
+    }
+
+    logger.warn(
+      {
+        rateLimitResetSeconds,
+        endpoint,
+        remainingRequests: getHeader(err.response.headers, 'ratelimit-remaining'),
+        resetTime: getHeader(err.response.headers, 'ratelimit-reset'),
+      },
+      'Rate limit details',
+    )
+
+    return new RateLimitError(rateLimitResetSeconds, endpoint, err)
+  }
+
+  // Check if this is a 403 error which can also indicate rate limiting
+  if (err && err.response && err.response.status === 403) {
+    const errorMessage = err.response.data?.message || err.message || ''
+    if (errorMessage.toLowerCase().includes('rate limit')) {
+      logger.warn('GitLab API rate limit exceeded (403)')
+      let rateLimitResetSeconds = 60
+
+      const retryAfter = getHeader(err.response.headers, 'retry-after')
+      if (retryAfter) {
+        rateLimitResetSeconds = parseInt(retryAfter, 10)
+      } else {
+        const rateLimitReset = getHeader(err.response.headers, 'ratelimit-reset')
+        if (rateLimitReset) {
+          const resetTimestamp = parseInt(rateLimitReset, 10)
+          const now = Math.floor(Date.now() / 1000)
+          rateLimitResetSeconds = Math.max(resetTimestamp - now, 0) + 5
+        }
+      }
+
+      return new RateLimitError(rateLimitResetSeconds, endpoint, err)
+    }
+  }
+
+  logger.error(err, `Error while calling GitLab API endpoint: ${endpoint}`)
+  return err
+}
+
+export const isApproachingRateLimit = (
+  headers: Record<string, string> | undefined,
+  threshold = 10,
+): boolean => {
+  if (!headers) return false
+
+  const remaining = getHeader(headers, 'ratelimit-remaining')
+  if (remaining) {
+    const remainingCount = parseInt(remaining, 10)
+    return remainingCount <= threshold
+  }
+
+  return false
+}
+
+export const getRateLimitInfo = (headers: Record<string, string> | undefined) => {
+  if (!headers) return null
+
+  const limit = getHeader(headers, 'ratelimit-limit')
+  const remaining = getHeader(headers, 'ratelimit-remaining')
+  const reset = getHeader(headers, 'ratelimit-reset')
+
+  return {
+    limit: limit ? parseInt(limit, 10) : null,
+    remaining: remaining ? parseInt(remaining, 10) : null,
+    reset: reset ? parseInt(reset, 10) : null,
+    resetDate: reset ? new Date(parseInt(reset, 10) * 1000) : null,
+  }
+}
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getForks.ts b/services/libs/integrations/src/integrations/gitlab/api/getForks.ts
@@ -5,6 +5,7 @@ import { GitlabForkData } from '../types'
 import { GitlabApiResult } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getForks = async ({
@@ -34,6 +35,8 @@ export const getForks = async ({
     forks = (await api.Projects.allForks(projectId, {
       updatedAfter: since,
     })) as ProjectSchema[]
+  } catch (error) {
+    throw handleGitlabError(error, `getForks:${projectId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getIssueDiscussions.ts b/services/libs/integrations/src/integrations/gitlab/api/getIssueDiscussions.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabDisccusionCommentData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getIssueDiscussions = async ({
@@ -40,6 +41,8 @@ export const getIssueDiscussions = async ({
 
     discussions = response.data as DiscussionSchema[]
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(error, `getIssueDiscussions:${projectId}:${issueIId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getIssues.ts b/services/libs/integrations/src/integrations/gitlab/api/getIssues.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabIssueData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getIssues = async ({
@@ -33,6 +34,7 @@ export const getIssues = async ({
   let issues: IssueSchema[]
 
   try {
+    await semaphore.acquire()
     const response = await api.Issues.all({
       projectId,
       page,
@@ -43,6 +45,8 @@ export const getIssues = async ({
 
     issues = response.data as IssueSchema[]
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(error, `getIssues:${projectId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestCommits.ts b/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestCommits.ts
@@ -6,6 +6,8 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabMergeRequestCommitData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
+
 export const getMergeRequestCommits = async ({
   api,
   projectId,
@@ -31,6 +33,7 @@ export const getMergeRequestCommits = async ({
   const extendedCommits: ExpandedCommitSchema[] = []
 
   try {
+    await semaphore.acquire()
     const response = await api.MergeRequests.allCommits(projectId, mergeRequestIId, {
       page,
       perPage: 20,
@@ -47,11 +50,26 @@ export const getMergeRequestCommits = async ({
         extendedCommits.push(extendedCommit as ExpandedCommitSchema)
         await timeout(500)
       } catch (error) {
+        const handledError = handleGitlabError(
+          error,
+          `getMergeRequestCommits:getCommit:${projectId}:${commit.id}`,
+          ctx.log,
+        )
+        // Only rethrow if it's a rate limit error
+        if (handledError.name === 'RateLimitError') {
+          throw handledError
+        }
         ctx.log.error(`Failed to fetch extended commit for ${commit.id}: ${error}`)
       }
     }
 
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(
+      error,
+      `getMergeRequestCommits:${projectId}:${mergeRequestIId}`,
+      ctx.log,
+    )
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestDiscussions.ts b/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestDiscussions.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabDisccusionCommentData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getMergeRequestDiscussions = async ({
@@ -33,6 +34,7 @@ export const getMergeRequestDiscussions = async ({
   let discussions: DiscussionSchema[]
 
   try {
+    await semaphore.acquire()
     const response = await api.MergeRequestDiscussions.all(projectId, mergeRequestIId, {
       showExpanded: true,
       page,
@@ -41,6 +43,12 @@ export const getMergeRequestDiscussions = async ({
 
     discussions = response.data as DiscussionSchema[]
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(
+      error,
+      `getMergeRequestDiscussions:${projectId}:${mergeRequestIId}`,
+      ctx.log,
+    )
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestDiscussionsAndEvents.ts b/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestDiscussionsAndEvents.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabDisccusionCommentData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getMergeRequestDiscussionsAndEvents = async ({
@@ -41,6 +42,12 @@ export const getMergeRequestDiscussionsAndEvents = async ({
 
     discussions = response.data as DiscussionSchema[]
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(
+      error,
+      `getMergeRequestDiscussionsAndEvents:${projectId}:${mergeRequestIId}`,
+      ctx.log,
+    )
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestEvents.ts b/services/libs/integrations/src/integrations/gitlab/api/getMergeRequestEvents.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabDisccusionCommentData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getMergeRequestEvents = async ({
@@ -41,6 +42,8 @@ export const getMergeRequestEvents = async ({
 
     discussions = response.data as DiscussionSchema[]
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(error, `getMergeRequestEvents:${projectId}:${mergeRequestIId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getMergeRequests.ts b/services/libs/integrations/src/integrations/gitlab/api/getMergeRequests.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabMergeRequestData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getMergeRequests = async ({
@@ -44,6 +45,8 @@ export const getMergeRequests = async ({
 
     mergeRequests = response.data as MergeRequestSchema[]
     pagination = response.paginationInfo
+  } catch (error) {
+    throw handleGitlabError(error, `getMergeRequests:${projectId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getStars.ts b/services/libs/integrations/src/integrations/gitlab/api/getStars.ts
@@ -4,6 +4,7 @@ import { IProcessStreamContext } from '../../../types'
 import { GitlabApiResult, GitlabStarData } from '../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
 import { getUser } from './getUser'
 
 export const getStars = async ({
@@ -28,6 +29,8 @@ export const getStars = async ({
   try {
     await semaphore.acquire()
     stars = (await api.Projects.allStarrers(projectId)) as ProjectStarrerSchema[]
+  } catch (error) {
+    throw handleGitlabError(error, `getStars:${projectId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/getUser.ts b/services/libs/integrations/src/integrations/gitlab/api/getUser.ts
@@ -3,6 +3,8 @@ import { Gitlab, SimpleUserSchema, UserSchema } from '@gitbeaker/rest'
 import type { IProcessStreamContext } from '../../../types'
 import { RedisSemaphore } from '../utils/lock'
 
+import { handleGitlabError } from './errorHandler'
+
 export const getUser = async (
   api: InstanceType<typeof Gitlab>,
   userId: number,
@@ -29,7 +31,7 @@ export const getUser = async (
     user = (await api.Users.show(userId)) as UserSchema
   } catch (error) {
     ctx.log.error(error, 'Failed to get user', userId)
-    throw error
+    throw handleGitlabError(error, `getUser:${userId}`, ctx.log)
   } finally {
     await semaphore.release()
   }
@@ -63,6 +65,8 @@ export const getUserByUsername = async (
   try {
     await semaphore.acquire()
     users = (await api.Search.all('users', username)) as SimpleUserSchema[]
+  } catch (error) {
+    throw handleGitlabError(error, `getUserByUsername:${username}`, ctx.log)
   } finally {
     await semaphore.release()
   }
diff --git a/services/libs/integrations/src/integrations/gitlab/api/refreshToken.ts b/services/libs/integrations/src/integrations/gitlab/api/refreshToken.ts
@@ -2,17 +2,24 @@ import axios from 'axios'
 
 import { IProcessStreamContext } from '../../../types'
 
+import { handleGitlabError } from './errorHandler'
+
 export const refreshToken = async (ctx: IProcessStreamContext) => {
   const platformSettings = ctx.platformSettings as {
     clientId: string
     clientSecret: string
   }
-  const response = await axios.post('https://gitlab.com/oauth/token', {
-    grant_type: 'refresh_token',
-    refresh_token: ctx.integration.refreshToken,
-    client_id: platformSettings.clientId,
-    client_secret: platformSettings.clientSecret,
-  })
 
-  return response.data
+  try {
+    const response = await axios.post('https://gitlab.com/oauth/token', {
+      grant_type: 'refresh_token',
+      refresh_token: ctx.integration.refreshToken,
+      client_id: platformSettings.clientId,
+      client_secret: platformSettings.clientSecret,
+    })
+
+    return response.data
+  } catch (error) {
+    throw handleGitlabError(error, 'refreshToken', ctx.log)
+  }
 }
