fix: fetch gitlab groups concurrently and improve logs (#4267)

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

backend/src/serverless/integrations/usecases/gitlab/getProjects.ts

@@ -23,37 +23,48 @@ export async function fetchAllGitlabGroups(accessToken: string) {
   }))
 }
 
-export async function fetchGitlabGroupProjects(accessToken: string, groups: any[]) {
-  const groupProjects = {}
+async function fetchProjectsForGroup(accessToken: string, group: any) {
+  const projects = []
+  let page = 1
+  let hasMorePages = true
 
-  for (const group of groups) {
-    const projects = []
-    let page = 1
-    let hasMorePages = true
+  while (hasMorePages) {
+    const response = await axios.get(`https://gitlab.com/api/v4/groups/${group.id}/projects`, {
+      headers: { Authorization: `Bearer ${accessToken}` },
+      params: { page, per_page: 100, archived: false },
+    })
+    projects.push(...response.data)
+    hasMorePages = response.headers['x-next-page'] !== ''
+    page++
+  }
 
-    while (hasMorePages) {
-      const response = await axios.get(`https://gitlab.com/api/v4/groups/${group.id}/projects`, {
-        headers: { Authorization: `Bearer ${accessToken}` },
-        params: { page, per_page: 100, archived: false },
-      })
-      projects.push(...response.data)
-      hasMorePages = response.headers['x-next-page'] !== ''
-      page++
-    }
+  return projects.map((project) => ({
+    groupId: group.id,
+    groupName: group.name,
+    groupPath: group.path,
+    id: project.id,
+    name: project.name,
+    path_with_namespace: project.path_with_namespace,
+    enabled: false,
+    forkedFrom: project?.forked_from_project?.web_url || null,
+  }))
+}
 
-    groupProjects[group.id] = projects.map((project) => ({
-      groupId: group.id,
-      groupName: group.name,
-      groupPath: group.path,
-      id: project.id,
-      name: project.name,
-      path_with_namespace: project.path_with_namespace,
-      enabled: false,
-      forkedFrom: project?.forked_from_project?.web_url || null,
-    }))
+export async function fetchGitlabGroupProjects(accessToken: string, groups: any[]) {
+  const CONCURRENCY = 10
+  const groupProjects: Record<number, any[]> = {}
+
+  for (let i = 0; i < groups.length; i += CONCURRENCY) {
+    const batch = groups.slice(i, i + CONCURRENCY)
+    const results = await Promise.all(
+      batch.map((group) => fetchProjectsForGroup(accessToken, group)),
+    )
+    batch.forEach((group, idx) => {
+      groupProjects[group.id] = results[idx]
+    })
   }
 
-  return groupProjects as Record<number, any[]>
+  return groupProjects
 }
 
 export async function fetchGitlabUserProjects(accessToken: string, userId: number) {

backend/src/services/integrationService.ts

@@ -2774,6 +2774,18 @@ export default class IntegrationService {
 
       await SequelizeRepository.commitTransaction(transaction)
     } catch (err) {
+      this.options.log.error(
+        {
+          errMessage: err?.message,
+          errName: err?.name,
+          errStack: err?.stack,
+          gitlabStatus: err?.response?.status,
+          gitlabError: err?.response?.data,
+          gitlabUrl: err?.config?.url,
+          gitlabMethod: err?.config?.method,
+        },
+        'gitlabConnect failed',
+      )
       await SequelizeRepository.rollbackTransaction(transaction)
       throw err
     }