Skip to content

Commit a318b59

Browse files
authored
feat: limit calendar fetch concurrency to 5 in getOrgDashboardData to prevent 429 rate limits (JhaSourav07#2150)
## Description Fixes JhaSourav07#2106 `getOrgDashboardData` was firing all member calendar fetches simultaneously via an uncapped `Promise.all`, creating a 1:1 ratio between org size and parallel HTTP requests. For large orgs (up to 200 members post-pagination fix), this risks Vercel serverless timeouts and GitHub secondary rate limits (429s). This PR introduces a dependency-free `runCappedConcurrency` helper that enforces a maximum of 5 concurrent requests at any time, draining the queue as slots free up. **Changes made:** - `lib/github.ts` — replaced uncapped `Promise.all` with `runCappedConcurrency(tasks, 5)`; appended the `runCappedConcurrency` helper at the end of the file - `lib/github.test.ts` — imported `runCappedConcurrency`; added 3 unit tests covering result order preservation, strict concurrency cap enforcement, and graceful per-task failure recovery ## Pillar - [ ] 🎨 Pillar 1 — New Theme Design - [ ] 📐 Pillar 2 — Geometric SVG Improvement - [ ] 🕐 Pillar 3 — Timezone Logic Optimization - [x] 🛠️ Other (Bug fix, refactoring, docs) ## Visual Preview N/A — internal data-fetching logic, no SVG output change. ## Checklist before requesting a review: - [x] I have read the `CONTRIBUTING.md` file. - [x] I have tested these changes locally (`localhost:3000/api/streak?user=Pranav-IIITM`). - [x] I have run `npm run format` and `npm run lint` locally and resolved all errors. - [x] My commits follow the Conventional Commits format (`feat: limit calendar fetch concurrency to 5 in getOrgDashboardData to prevent 429 rate limits`). - [ ] I have updated `README.md` if I added a new theme or URL parameter. *(not applicable)* - [x] I have started the repo. - [x] I have made sure that i have only one commit to merge in this PR. - [x] The SVG output matches the CommitPulse "premium quality" aesthetic standard. *(not applicable — no SVG changes)* - [x] (Recommended) I joined the CommitPulse Discord community for contributor discussions, mentorship, and faster PR support.
2 parents cd029a3 + 0a6eee5 commit a318b59

2 files changed

Lines changed: 83 additions & 8 deletions

File tree

lib/github.test.ts

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
getOrgDashboardData,
1717
getWrappedData,
1818
computeDeveloperScore,
19+
runCappedConcurrency,
1920
buildProfileData,
2021
aggregateLanguages,
2122
buildInsights,
@@ -2106,3 +2107,44 @@ describe('computeDeveloperScore', () => {
21062107
).toBe(2);
21072108
});
21082109
});
2110+
2111+
describe('runCappedConcurrency', () => {
2112+
it('should process all items and return their results in order', async () => {
2113+
const items = [1, 2, 3, 4, 5];
2114+
const results = await runCappedConcurrency(items, 2, async (x) => {
2115+
return x * 10;
2116+
});
2117+
2118+
expect(results).toEqual([10, 20, 30, 40, 50]);
2119+
});
2120+
2121+
it('should strictly limit the concurrency to the specified limit', async () => {
2122+
const items = [1, 2, 3, 4, 5];
2123+
let activePromises = 0;
2124+
let maxConcurrentPromises = 0;
2125+
2126+
await runCappedConcurrency(items, 2, async (x) => {
2127+
activePromises++;
2128+
maxConcurrentPromises = Math.max(maxConcurrentPromises, activePromises);
2129+
await new Promise((resolve) => setTimeout(resolve, 5));
2130+
activePromises--;
2131+
return x;
2132+
});
2133+
2134+
expect(maxConcurrentPromises).toBeLessThanOrEqual(2);
2135+
});
2136+
2137+
it('should handle errors inside tasks gracefully without aborting the entire sequence', async () => {
2138+
const items = [1, 2, 3, 4, 5];
2139+
const results = await runCappedConcurrency(items, 2, async (x) => {
2140+
if (x === 3) throw new Error('Task 3 failed');
2141+
return x * 10;
2142+
});
2143+
2144+
expect(results[0]).toBe(10);
2145+
expect(results[1]).toBe(20);
2146+
expect(results[2]).toBeNull();
2147+
expect(results[3]).toBe(40);
2148+
expect(results[4]).toBe(50);
2149+
});
2150+
});

lib/github.ts

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -710,17 +710,18 @@ export async function getOrgDashboardData(orgName: string, options: FetchOptions
710710
throw new Error('This endpoint is strictly for organizations.');
711711
if (membersOrError instanceof Error) throw membersOrError;
712712

713-
const members: string[] = membersOrError;
713+
const members = membersOrError;
714+
715+
// Fetch calendars for all members concurrently with capped concurrency to avoid 429s/timeouts
714716
const calendars = (
715-
await Promise.all(
716-
members.map((m) =>
717-
fetchGitHubContributions(m, options)
718-
.then((d) => d.calendar)
719-
.catch(() => null)
720-
)
717+
await runCappedConcurrency(members, 5, (member) =>
718+
fetchGitHubContributions(member, options)
719+
.then((data) => data.calendar)
720+
.catch(() => null)
721721
)
722-
).filter((c): c is ContributionCalendar => c !== null);
722+
).filter((c: ContributionCalendar | null) => c !== null) as ContributionCalendar[];
723723

724+
// Create the Mega-City
724725
const aggregatedCalendar = aggregateCalendars(calendars);
725726
const streakStats = calculateStreak(aggregatedCalendar);
726727
const totalStars = reposData.reduce((acc, r) => acc + r.stargazers_count, 0);
@@ -1225,3 +1226,35 @@ export async function getWrappedData(
12251226
topLanguage,
12261227
};
12271228
}
1229+
1230+
/**
1231+
* Run tasks concurrently with a maximum limit on active promises.
1232+
*/
1233+
export async function runCappedConcurrency<T, R>(
1234+
items: T[],
1235+
limit: number,
1236+
fn: (item: T) => Promise<R>
1237+
): Promise<R[]> {
1238+
const results: R[] = new Array(items.length);
1239+
let currentIndex = 0;
1240+
1241+
async function worker(): Promise<void> {
1242+
while (currentIndex < items.length) {
1243+
const index = currentIndex++;
1244+
try {
1245+
results[index] = await fn(items[index]);
1246+
} catch (err) {
1247+
results[index] = null as unknown as R;
1248+
}
1249+
}
1250+
}
1251+
1252+
const workers: Promise<void>[] = [];
1253+
const workerCount = Math.min(limit, items.length);
1254+
for (let i = 0; i < workerCount; i++) {
1255+
workers.push(worker());
1256+
}
1257+
1258+
await Promise.all(workers);
1259+
return results;
1260+
}

0 commit comments

Comments
 (0)