fix: store topUsers in batches

Author: nikohoffren

utils/fetchTopUsersByPullRequests.ts

@@ -9,58 +9,98 @@ type UserStat = {
 export const fetchTopUsersByPullRequests = async (
   repo: string
 ): Promise<UserStat[]> => {
-  //* Limit to first 300 PRs only
-  let url = `https://api.github.com/repos/${repo}/pulls?state=closed&per_page=100&page=1`;
   const userStats: { [key: string]: { prCount: number; avatarUrl: string } } = {};
-  let pageCount = 0;
-  const MAX_PAGES = 3;
+  const MAX_CONCURRENT_REQUESTS = 3;
+  const PER_PAGE = 100;
 
-  while (url && pageCount < MAX_PAGES) {
-    const response = await axios.get(url, {
-      timeout: 5000,
+  try {
+    // First, get the total PR count
+    const initialResponse = await axios.get(`https://api.github.com/repos/${repo}/pulls`, {
+      params: {
+        state: 'closed',
+        per_page: 1
+      },
       headers: {
         Authorization: `token ${process.env.GITHUB_TOKEN}`,
       }
     });
-    pageCount++;
-    const prData = response.data;
-    prData.forEach((pr: any) => {
-      if (pr.merged_at) {
-        const username = pr.user.login;
 
-        //* Ignore dependabot and nikohoffren PRs
-        if (
-          username === "dependabot" ||
-          username === "dependabot[bot]" ||
-          username === "nikohoffren"
-        ) {
-          return;
-        }
+    const linkHeader = initialResponse.headers.link;
+    const lastPageMatch = linkHeader?.match(/page=(\d+)>; rel="last"/);
+    const totalPages = lastPageMatch ? parseInt(lastPageMatch[1]) : 1;
+
+    // Fetch PRs in batches
+    for (let i = 0; i < totalPages; i += MAX_CONCURRENT_REQUESTS) {
+      const requests = [];
 
-        const avatarUrl = pr.user.avatar_url;
-        userStats[username] = {
-          prCount: (userStats[username]?.prCount || 0) + 1,
-          avatarUrl,
-        };
+      // Create batch of concurrent requests
+      for (let j = 0; j < MAX_CONCURRENT_REQUESTS && (i + j) < totalPages; j++) {
+        const pageNum = i + j + 1;
+        requests.push(
+          axios.get(`https://api.github.com/repos/${repo}/pulls`, {
+            params: {
+              state: 'closed',
+              per_page: PER_PAGE,
+              page: pageNum
+            },
+            headers: {
+              Authorization: `token ${process.env.GITHUB_TOKEN}`,
+            },
+            timeout: 8000
+          }).catch(error => {
+            console.error(`Failed to fetch page ${pageNum}:`, error);
+            return { data: [] };
+          })
+        );
       }
-    });
 
-    const linkHeader = response.headers.link;
-    const nextLink = linkHeader
-      ? linkHeader.split(",").find((s: string) => s.includes('rel="next"'))
-      : null;
-    url = nextLink ? nextLink.match(/<(.*)>/)?.[1] : null;
-  }
+      // Wait for batch to complete
+      const responses = await Promise.all(requests);
 
-  const sortedUsers: UserStat[] = Object.entries(userStats)
-    .sort(([, a], [, b]) => b.prCount - a.prCount)
-    .slice(0, 20)
-    .map(([username, { prCount, avatarUrl }]) => ({
-      username,
-      prCount,
-      avatarUrl,
-    }));
+      // Process the responses
+      responses.forEach(response => {
+        if (response.data) {
+          response.data.forEach((pr: any) => {
+            if (pr.merged_at) {
+              const username = pr.user.login;
 
-  return sortedUsers;
-};
+              // Ignore specific users
+              if (
+                username === "dependabot" ||
+                username === "dependabot[bot]" ||
+                username === "nikohoffren"
+              ) {
+                return;
+              }
+
+              const avatarUrl = pr.user.avatar_url;
+              userStats[username] = {
+                prCount: (userStats[username]?.prCount || 0) + 1,
+                avatarUrl,
+              };
+            }
+          });
+        }
+      });
+
+      // Add a small delay between batches to avoid rate limiting
+      if (i + MAX_CONCURRENT_REQUESTS < totalPages) {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+    }
 
+    const sortedUsers: UserStat[] = Object.entries(userStats)
+      .sort(([, a], [, b]) => b.prCount - a.prCount)
+      .slice(0, 20)
+      .map(([username, { prCount, avatarUrl }]) => ({
+        username,
+        prCount,
+        avatarUrl,
+      }));
+
+    return sortedUsers;
+  } catch (error) {
+    console.error('Error fetching PRs:', error);
+    throw error;
+  }
+};

utils/fetchTopUsersFromDb.ts

@@ -1,5 +1,12 @@
 import { connectToDB } from './db';
 
+interface UserStat {
+  login: string;
+  avatar_url: string;
+  html_url: string;
+  contributions: number;
+}
+
 export async function getTopUsersFromDb() {
   try {
     const { db } = await connectToDB();
@@ -11,18 +18,19 @@ export async function getTopUsersFromDb() {
   }
 }
 
-export async function storeTopUsersInDb(data: any[]) {
+export async function storeTopUsersInDb(data: UserStat[]) {
   try {
     const { db } = await connectToDB();
     await db.collection('topUsers').deleteMany({});
     await db.collection('topUsers').insertMany(
       data.map(user => ({
         ...user,
-        timestamp: new Date()
+        timestamp: new Date(),
+        lastUpdated: new Date().toISOString()
       }))
     );
   } catch (error) {
-    console.error('DB Storage Error:', error);
+    console.error('Error storing top users:', error);
     throw error;
   }
 }
@@ -56,3 +64,4 @@ export async function storeTopThreeUsersInDb(data: any[]) {
 
 
 
+