feat: add location detection for GitHub users and initialize PostgreSQL schema

- Enhanced GitHub user data model to include `login` and `location` fields.
- Implemented a location detector to normalize user-provided locations into country slugs.
- Updated GraphQL queries to fetch the new `login` and `location` fields.
- Increased default pull request count from 250 to 300.
- Added a script to initialize the PostgreSQL database schema.
- Updated package dependencies, including adding `pg` for PostgreSQL support.
- Refactored error handling and improved code formatting for better readability.

Co-authored-by: Copilot <copilot@github.com>

Author: O2sa

app/api/calculate-leaderboard/route.ts

@@ -3,6 +3,8 @@ import yaml from "js-yaml";
 import { fetchGitHubUserData } from "@/lib/github";
 import { calculateUserScore } from "@/lib/score";
 import { createCacheStore, getCacheConfigFromEnv } from "@/lib/cache-store";
+import { getDatabaseStore } from "@/lib/db-store";
+import { detectCountry } from "@/lib/location-detector";
 
 export const runtime = "nodejs";
 
@@ -48,6 +50,25 @@ function buildLeaderboardCacheKey(country: string, namespace: string): string {
   return `${namespace}:leaderboard:${country.trim().toLowerCase()}`;
 }
 
+function getEnvInt(key: string, fallback: number): number {
+  const raw = process.env[key]?.trim();
+  if (!raw) return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function getStaleDays(): number {
+  return getEnvInt("LEADERBOARD_USER_STALE_DAYS", 30);
+}
+
+function getSeedLimit(): number {
+  return getEnvInt("LEADERBOARD_SEED_LIMIT", 256);
+}
+
+function getRefreshLimit(): number {
+  return getEnvInt("LEADERBOARD_REFRESH_LIMIT", 500);
+}
+
 // ─── POST handler ──────────────────────────────────────────────────────
 
 export async function POST(request: Request) {
@@ -61,6 +82,10 @@ export async function POST(request: Request) {
     );
   }
 
+  // Ensure database schema exists
+  const db = getDatabaseStore();
+  await db.initializeSchema();
+
   // ── 1. Fetch committers from committers.top ──────────────────────────
   let committersData: {
     title: string;
@@ -110,63 +135,144 @@ export async function POST(request: Request) {
     );
   }
 
-  // ── 2. Calculate scores for all users ────────────────────────────────
-  const scored: ScoredEntry[] = [];
+  // ── 2a. Seed NEW users from committers.top ───────────────────────────
   const errors: string[] = [];
+  const seedLimit = getSeedLimit();
+  const usersToSeed = committersData.users.slice(0, seedLimit);
+  let newUsersCount = 0;
+  let skippedExistingCount = 0;
 
-  // Build rank lookup from original committers data
-  const rankMap = new Map(
-    committersData.users.map((u) => [
-      u.login,
-      { rank: u.rank, contributions: u.contributions },
-    ]),
-  );
-
-  for (const user of committersData.users) {
+  for (const user of usersToSeed) {
     try {
+      const exists = await db.userExists(user.login);
+      if (exists) {
+        skippedExistingCount += 1;
+        continue;
+      }
+
       const data = await fetchGitHubUserData(user.login);
       const score = calculateUserScore(data, user.login);
-      const original = rankMap.get(user.login) ?? { rank: 0, contributions: 0 };
-      scored.push({
-        username: user.login,
+      const countryDetected = detectCountry(data.location);
+
+      await db.upsertUser({
+        username: data.login,
         name: data.name,
         avatarUrl: data.avatarUrl,
+        location: data.location,
+        country: countryDetected,
+        rawData: data,
+        scores: score,
         repoScore: Math.round(score.repoScore),
         prScore: Math.round(score.prScore),
         contributionScore: Math.round(score.contributionScore),
         finalScore: Math.round(score.finalScore),
-        originalRank: original.rank,
-        originalContributions: original.contributions,
-        impactRank: 0,
+        staleDays: getStaleDays(),
       });
+
+      newUsersCount += 1;
     } catch {
       errors.push(user.login);
     }
   }
 
-  // Sort by finalScore descending and assign impactRank
+  // ── 2b. Refresh STALE users from DB top N ───────────────────────────
+  const refreshLimit = getRefreshLimit();
+  const topUsers = await db.getTopUsers(country, refreshLimit);
+  let refreshedCount = 0;
+
+  for (const row of topUsers) {
+    // Check if stale
+    if (row.stale_after >= new Date()) {
+      continue; // still fresh
+    }
+
+    try {
+      const data = await fetchGitHubUserData(row.username);
+      const score = calculateUserScore(data, row.username);
+      const countryDetected = detectCountry(data.location);
+
+      await db.upsertUser({
+        username: data.login,
+        name: data.name,
+        avatarUrl: data.avatarUrl,
+        location: data.location,
+        country: countryDetected,
+        rawData: data,
+        scores: score,
+        repoScore: Math.round(score.repoScore),
+        prScore: Math.round(score.prScore),
+        contributionScore: Math.round(score.contributionScore),
+        finalScore: Math.round(score.finalScore),
+        staleDays: getStaleDays(),
+      });
+
+      refreshedCount += 1;
+    } catch {
+      errors.push(row.username);
+    }
+  }
+
+  // ── 3. Build & cache the leaderboard result ──────────────────────────
+  const allUsers = await db.getLeaderboard(country, getEnvInt("LEADERBOARD_DISPLAY_LIMIT", 500));
+  const totalCount = await db.getLeaderboardCount(country);
+
+  // Build rank lookup from original committers data
+  const rankMap = new Map(
+    committersData.users.map((u) => [
+      u.login,
+      { rank: u.rank, contributions: u.contributions },
+    ]),
+  );
+
+  const scored: ScoredEntry[] = allUsers.map((row) => {
+    const original = rankMap.get(row.username) ?? { rank: 0, contributions: 0 };
+    return {
+      username: row.username,
+      name: row.name,
+      avatarUrl: row.avatar_url,
+      repoScore: row.repo_score,
+      prScore: row.pr_score,
+      contributionScore: row.contribution_score,
+      finalScore: row.final_score,
+      originalRank: original.rank,
+      originalContributions: original.contributions,
+      impactRank: 0,
+    };
+  });
+
+  // Sort and assign impactRank
   scored.sort((a, b) => b.finalScore - a.finalScore);
   scored.forEach((u, idx) => (u.impactRank = idx + 1));
 
   const result: LeaderboardResult = {
     title: committersData.title,
-    totalFromSource: committersData.totalFromSource,
+    totalFromSource: totalCount,
     scored,
-    errors,
+    errors: [...new Set(errors)], // deduplicate
   };
 
-  // ── 3. Store in Redis cache ──────────────────────────────────────────
+  // ── 4. Invalidate Redis cache (lazy rebuild on next GET) ─────────────
   try {
     const cacheConfig = getCacheConfigFromEnv();
     const cacheStore = createCacheStore(cacheConfig);
-    if (cacheStore.enabled) {
+    if (cacheStore.enabled && cacheStore.del) {
       const cacheKey = buildLeaderboardCacheKey(country, cacheConfig.namespace);
-      await cacheStore.set(cacheKey, result, cacheConfig.ttlSeconds);
+      await cacheStore.del(cacheKey);
     }
   } catch {
-    // Non-fatal: cache write failure should not block the response
-    console.warn("Failed to cache leaderboard result");
+    // Non-fatal: cache invalidation failure should not block the response
+    console.warn("Failed to invalidate leaderboard cache");
   }
 
-  return NextResponse.json({ success: true, ...result });
-}
+  return NextResponse.json({
+    success: true,
+    ...result,
+    _meta: {
+      newUsers: newUsersCount,
+      refreshedUsers: refreshedCount,
+      skippedExisting: skippedExistingCount,
+      errors: errors.length,
+      totalInDb: totalCount,
+    },
+  });
+}

app/api/compare/route.ts

@@ -3,6 +3,12 @@ import { fetchGitHubUserData } from "../../../lib/github";
 import { calculateUserScore } from "../../../lib/score";
 import { normalizeSelectedLanguages } from "@/lib/scoring/languageScoring";
 import { toSafeApiError } from "@/lib/github-graphql-client";
+import { getDatabaseStore } from "@/lib/db-store";
+import {
+  createCacheStore,
+  getCacheConfigFromEnv,
+} from "@/lib/cache-store";
+import { detectCountry } from "@/lib/location-detector";
 import type { CompareInsights, SafeApiError } from "@/types/api-response";
 import {
   DEFAULT_LOCALE,
@@ -322,7 +328,7 @@ async function compareUsers(
     );
 
     results.push({
-      username,
+      username: data.login,
       name: data.name,
       avatarUrl: data.avatarUrl,
       repoScore: Math.round(score.repoScore),
@@ -340,6 +346,43 @@ async function compareUsers(
       signals: score.signals,
       explanations: score.explanations,
     });
+
+    // ── Fire-and-forget: detect country & upsert into DB ──────────────
+    const country = detectCountry(data.location);
+    if (country) {
+      const staleDays = parseInt(
+        process.env.GITHUB_USER_STALE_DAYS ?? "14",
+        10,
+      );
+
+      const db = getDatabaseStore();
+      db.upsertUser({
+        username: data.login,
+        name: data.name,
+        avatarUrl: data.avatarUrl,
+        location: data.location,
+        country,
+        rawData: data,
+        scores: score,
+        repoScore: Math.round(score.repoScore),
+        prScore: Math.round(score.prScore),
+        contributionScore: Math.round(score.contributionScore),
+        finalScore: Math.round(score.finalScore),
+        staleDays,
+      })
+        .then(() => {
+          // Invalidate Redis cache for this country
+          const cacheConfig = getCacheConfigFromEnv();
+          const cacheStore = createCacheStore(cacheConfig);
+          if (cacheStore.enabled && cacheStore.del) {
+            const key = `${cacheConfig.namespace}:leaderboard:${country}`;
+            cacheStore.del(key).catch(() => {});
+          }
+        })
+        .catch((err: unknown) => {
+          console.warn("Failed to upsert user from compare:", err);
+        });
+    }
   }
 
   return results;