Skip to content

Commit f9aecb8

Browse files
KristjanESPERANTOKristjanESPERANTO
committed
fix: harden GraphQL fallback and split GitHub commit-call metrics
1 parent 06000ac commit f9aecb8

2 files changed

Lines changed: 179 additions & 26 deletions

File tree

‎scripts/collect-metadata/index.ts‎

Lines changed: 132 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
/* Unified metadata collector for stage 1+2 */
22

3-
import { fetchRepositoryData, normalizeRepositoryData } from "../updateRepositoryApiData/api.ts";
3+
import {
4+
fetchRepositoryData,
5+
getRepositoryApiMetrics,
6+
normalizeRepositoryData,
7+
resetRepositoryApiMetrics
8+
} from "../updateRepositoryApiData/api.ts";
49
import type { NormalizedRepositoryMetadata } from "../updateRepositoryApiData/api.ts";
510
import { getRepositoryId, getRepositoryType } from "../updateRepositoryApiData/helpers.ts";
611
import { createHttpClient } from "../shared/http-client.ts";
@@ -69,6 +74,12 @@ interface ModuleStats {
6974
misses: number;
7075
}
7176

77+
interface GitHubBatchDiagnostics {
78+
individualFallbackAttempts: number;
79+
individualFallbackSuccesses: number;
80+
splitRetries: number;
81+
}
82+
7283
interface CircuitBreaker {
7384
consecutive403Errors: number;
7485
stopFetching: boolean;
@@ -95,10 +106,12 @@ interface IndividualFetchFailure {
95106
}
96107

97108
type IndividualFetchResult = IndividualFetchSuccess | IndividualFetchFailure;
109+
type IndividualFetchSource = "direct" | "batch-recovery";
98110

99111
interface AppendGitHubBatchResultOptions {
100112
batchFailed: boolean;
101113
batchResults: Record<string, GraphQlRepoData>;
114+
diagnostics: GitHubBatchDiagnostics;
102115
enrichedModules: EnrichedModule[];
103116
module: EnrichedModule;
104117
previousModulesMap: Map<string, EnrichedModule>;
@@ -132,6 +145,7 @@ const WIKI_URL = "https://raw.githubusercontent.com/wiki/MagicMirrorOrg/MagicMir
132145
const REPOSITORY_CACHE_PATH = path.join("website", "data", "cache", "repository-api-cache.json");
133146
const REPOSITORY_CACHE_TTL_MS = 1000 * 60 * 60 * 24 * 3; // 3 days
134147
const GITHUB_GRAPHQL_BATCH_SIZE = 100;
148+
const GITHUB_GRAPHQL_MIN_BATCH_SIZE = 20;
135149
const FORCE_REFRESH = process.env.FORCE_REFRESH === "true";
136150

137151
const rateLimiter = createRateLimiter({
@@ -297,9 +311,16 @@ function getFallbackOrOriginal(module: EnrichedModule, previousModulesMap: Map<s
297311
/**
298312
* Helper to fetch individual repository data.
299313
*/
300-
async function fetchIndividualModule(module: EnrichedModule, repoType: string, client: HttpClient): Promise<IndividualFetchResult> {
314+
async function fetchIndividualModule(
315+
module: EnrichedModule,
316+
repoType: string,
317+
client: HttpClient,
318+
source: IndividualFetchSource
319+
): Promise<IndividualFetchResult> {
301320
try {
302-
const { response, data, branchData } = await fetchRepositoryData(module, client);
321+
const { response, data, branchData } = await fetchRepositoryData(module, client, process.env, {
322+
githubCommitCallSource: repoType === "github" ? source : undefined
323+
});
303324

304325
if (response && !response.ok) {
305326
throw new Error(`API Error: ${response.status} ${response.statusText || ""}`);
@@ -342,6 +363,7 @@ async function appendGitHubBatchResult({
342363
module,
343364
batchResults,
344365
batchFailed,
366+
diagnostics,
345367
previousModulesMap,
346368
stats,
347369
enrichedModules
@@ -371,10 +393,15 @@ async function appendGitHubBatchResult({
371393
logger.warn(`No data returned for ${module.name} in batch`, { url: module.url });
372394
}
373395

374-
const recovery = await fetchIndividualModule(module, "github", httpClient);
396+
if (batchFailed) {
397+
diagnostics.individualFallbackAttempts += 1;
398+
}
399+
400+
const recovery = await fetchIndividualModule(module, "github", httpClient, "batch-recovery");
375401
if (recovery.success) {
376402
const normalized = recovery.data;
377403
if (batchFailed) {
404+
diagnostics.individualFallbackSuccesses += 1;
378405
logger.info(`Recovered ${module.name} via individual GitHub fetch after batch failure`);
379406
}
380407
repositoryCache.set(repoId, normalized);
@@ -396,6 +423,89 @@ async function appendGitHubBatchResult({
396423
stats.errors += 1;
397424
}
398425

426+
function shouldRetryBatchWithSmallerChunks(batchFailed: boolean, batchError: string | null, chunkSize: number): boolean {
427+
if (!batchFailed || typeof batchError !== "string") {
428+
return false;
429+
}
430+
431+
return batchError.toLowerCase().includes("terminated") && chunkSize > GITHUB_GRAPHQL_MIN_BATCH_SIZE;
432+
}
433+
434+
async function processGitHubChunk({
435+
chunk,
436+
chunkStart,
437+
diagnostics,
438+
previousModulesMap,
439+
stats,
440+
enrichedModules
441+
}: {
442+
chunk: EnrichedModule[];
443+
chunkStart: number;
444+
diagnostics: GitHubBatchDiagnostics;
445+
previousModulesMap: Map<string, EnrichedModule>;
446+
stats: ModuleStats;
447+
enrichedModules: EnrichedModule[];
448+
}): Promise<void> {
449+
const { results: batchResults, failed: batchFailed, error: batchError } = await fetchGitHubBatch(chunk);
450+
451+
if (shouldRetryBatchWithSmallerChunks(batchFailed, batchError, chunk.length)) {
452+
diagnostics.splitRetries += 1;
453+
const splitPoint = Math.ceil(chunk.length / 2);
454+
const leftChunk = chunk.slice(0, splitPoint);
455+
const rightChunk = chunk.slice(splitPoint);
456+
457+
logger.warn("GitHub GraphQL chunk terminated, retrying with smaller batches", {
458+
chunkStart,
459+
chunkSize: chunk.length,
460+
leftChunkSize: leftChunk.length,
461+
rightChunkSize: rightChunk.length,
462+
error: batchError
463+
});
464+
465+
await processGitHubChunk({
466+
chunk: leftChunk,
467+
chunkStart,
468+
diagnostics,
469+
previousModulesMap,
470+
stats,
471+
enrichedModules
472+
});
473+
474+
if (rightChunk.length > 0) {
475+
await processGitHubChunk({
476+
chunk: rightChunk,
477+
chunkStart: chunkStart + splitPoint,
478+
diagnostics,
479+
previousModulesMap,
480+
stats,
481+
enrichedModules
482+
});
483+
}
484+
485+
return;
486+
}
487+
488+
if (batchFailed) {
489+
logger.warn("GitHub GraphQL chunk failed, retrying repositories individually", {
490+
chunkStart,
491+
chunkSize: chunk.length,
492+
error: batchError
493+
});
494+
}
495+
496+
for (const module of chunk) {
497+
await appendGitHubBatchResult({
498+
module,
499+
batchResults,
500+
batchFailed,
501+
diagnostics,
502+
previousModulesMap,
503+
stats,
504+
enrichedModules
505+
});
506+
}
507+
}
508+
399509
/**
400510
* Process a single module: check cache, queue for batch, or fetch individually.
401511
* Returns the enriched module or null if queued for batch processing.
@@ -446,7 +556,7 @@ async function processModule(module: EnrichedModule, context: ProcessModuleConte
446556
}
447557

448558
// Fetch individual (non-GitHub or no token)
449-
const recovery = await fetchIndividualModule(module, repoType, client);
559+
const recovery = await fetchIndividualModule(module, repoType, client, "direct");
450560

451561
if (recovery.success) {
452562
circuitBreaker.recordSuccess();
@@ -498,6 +608,11 @@ async function enrichModules(modules: ParsedModuleEntry[], previousModulesMap: M
498608
const enrichedModules: EnrichedModule[] = [];
499609
const githubModulesToFetch: EnrichedModule[] = [];
500610
const stats: ModuleStats = { hits: 0, misses: 0, errors: 0, fallbacks: 0 };
611+
const githubBatchDiagnostics: GitHubBatchDiagnostics = {
612+
splitRetries: 0,
613+
individualFallbackAttempts: 0,
614+
individualFallbackSuccesses: 0
615+
};
501616
let processedCount = 0;
502617
const totalModules = modules.length;
503618

@@ -545,27 +660,17 @@ async function enrichModules(modules: ParsedModuleEntry[], previousModulesMap: M
545660
logger.info(`Batch fetching ${githubModulesToFetch.length} GitHub repositories...`);
546661
for (let index = 0; index < githubModulesToFetch.length; index += GITHUB_GRAPHQL_BATCH_SIZE) {
547662
const chunk = githubModulesToFetch.slice(index, index + GITHUB_GRAPHQL_BATCH_SIZE);
548-
const { results: batchResults, failed: batchFailed, error: batchError } = await fetchGitHubBatch(chunk);
549-
550-
if (batchFailed) {
551-
logger.warn("GitHub GraphQL chunk failed, retrying repositories individually", {
552-
chunkStart: index,
553-
chunkSize: chunk.length,
554-
error: batchError
555-
});
556-
}
557-
558-
for (const module of chunk) {
559-
await appendGitHubBatchResult({
560-
module,
561-
batchResults,
562-
batchFailed,
563-
previousModulesMap,
564-
stats,
565-
enrichedModules
566-
});
567-
}
663+
await processGitHubChunk({
664+
chunk,
665+
chunkStart: index,
666+
diagnostics: githubBatchDiagnostics,
667+
previousModulesMap,
668+
stats,
669+
enrichedModules
670+
});
568671
}
672+
673+
logger.info("GitHub batch diagnostics", githubBatchDiagnostics);
569674
}
570675

571676
logger.info("Metadata collection stats", stats);
@@ -587,6 +692,7 @@ export async function runCollectMetadata({
587692
previousModulesMap = loadPreviousModules<EnrichedModule>()
588693
}: RunCollectMetadataOptions = {}): Promise<RunCollectMetadataResult> {
589694
logger.info("Starting unified metadata collection...");
695+
resetRepositoryApiMetrics();
590696

591697
if (!process.env.GITHUB_TOKEN) {
592698
logger.warn("GITHUB_TOKEN is not set. Rate limits will be strict and processing may be slow.");
@@ -621,6 +727,7 @@ export async function runCollectMetadata({
621727
}
622728

623729
await repositoryCache.flush();
730+
logger.info("Repository API diagnostics", getRepositoryApiMetrics());
624731
logger.info(`Collected ${enrichedModules.length} modules.`);
625732

626733
return {

‎scripts/updateRepositoryApiData/api.ts‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,12 @@ interface HttpClientLike {
2424
getJson: <TData = unknown>(url: string, options?: { headers?: RequestHeaders }) => Promise<HttpResponse<TData>>;
2525
}
2626

27+
type GitHubCommitCallSource = "direct" | "batch-recovery";
28+
29+
interface FetchRepositoryDataOptions {
30+
githubCommitCallSource?: GitHubCommitCallSource;
31+
}
32+
2733
interface RepositoryApiData {
2834
archived?: boolean;
2935
default_branch?: string;
@@ -75,7 +81,38 @@ export interface NormalizedRepositoryMetadata {
7581
stars: number;
7682
}
7783

78-
export async function fetchRepositoryData(module: RepositoryModuleRef, httpClient: HttpClientLike, env: NodeJS.ProcessEnv = process.env): Promise<FetchRepositoryDataResult> {
84+
interface RepositoryApiMetricsSnapshot {
85+
githubBranchCommitApiCallsBatchRecovery: number;
86+
githubBranchCommitApiCallsDirect: number;
87+
githubBranchCommitApiCalls: number;
88+
}
89+
90+
const repositoryApiMetrics: RepositoryApiMetricsSnapshot = {
91+
githubBranchCommitApiCallsBatchRecovery: 0,
92+
githubBranchCommitApiCallsDirect: 0,
93+
githubBranchCommitApiCalls: 0
94+
};
95+
96+
export function resetRepositoryApiMetrics(): void {
97+
repositoryApiMetrics.githubBranchCommitApiCallsBatchRecovery = 0;
98+
repositoryApiMetrics.githubBranchCommitApiCallsDirect = 0;
99+
repositoryApiMetrics.githubBranchCommitApiCalls = 0;
100+
}
101+
102+
export function getRepositoryApiMetrics(): RepositoryApiMetricsSnapshot {
103+
return {
104+
githubBranchCommitApiCallsBatchRecovery: repositoryApiMetrics.githubBranchCommitApiCallsBatchRecovery,
105+
githubBranchCommitApiCallsDirect: repositoryApiMetrics.githubBranchCommitApiCallsDirect,
106+
githubBranchCommitApiCalls: repositoryApiMetrics.githubBranchCommitApiCalls
107+
};
108+
}
109+
110+
export async function fetchRepositoryData(
111+
module: RepositoryModuleRef,
112+
httpClient: HttpClientLike,
113+
env: NodeJS.ProcessEnv = process.env,
114+
options: FetchRepositoryDataOptions = {}
115+
): Promise<FetchRepositoryDataResult> {
79116
const repoType = getRepositoryType(module.url);
80117
const repoId = getRepositoryId(module.url);
81118

@@ -144,6 +181,15 @@ export async function fetchRepositoryData(module: RepositoryModuleRef, httpClien
144181
}
145182

146183
if (branchUrl) {
184+
if (repoType === "github") {
185+
repositoryApiMetrics.githubBranchCommitApiCalls += 1;
186+
if (options.githubCommitCallSource === "batch-recovery") {
187+
repositoryApiMetrics.githubBranchCommitApiCallsBatchRecovery += 1;
188+
}
189+
else {
190+
repositoryApiMetrics.githubBranchCommitApiCallsDirect += 1;
191+
}
192+
}
147193
const branchResult = await httpClient.getJson<RepositoryBranchData>(branchUrl, { headers });
148194
branchData = branchResult.data;
149195
}

0 commit comments

Comments
 (0)