feat(sync): harden supabase queue on vercel, add chunked backfill UX, and switch commit matching to PAT identity

Author: marcus

src/app/api/onboarding/author-emails/route.ts

@@ -6,14 +6,20 @@ export async function POST(request: Request) {
   const { appUser } = await requireAppUser();
   const form = await request.formData();
   const raw = String(form.get("emails") ?? "");
+  const providerRaw = String(form.get("provider") ?? "").trim();
 
   const emails = raw
     .split(",")
     .map((email) => email.trim().toLowerCase())
     .filter(Boolean);
 
+  const provider =
+    providerRaw === "GITLAB" || providerRaw === "AZURE_DEVOPS" || providerRaw === "GITHUB"
+      ? providerRaw
+      : undefined;
+
   await prisma.integration.updateMany({
-    where: { userId: appUser.id },
+    where: provider ? { userId: appUser.id, provider } : { userId: appUser.id },
     data: { authorEmails: emails },
   });
 

src/app/dashboard/page.tsx

@@ -59,7 +59,7 @@ export default async function DashboardPage({
     }),
     prisma.manualHighlight.findMany({ where: { userId: appUser.id }, orderBy: { date: "desc" }, take: 20 }),
     prisma.publicShare.findMany({ where: { userId: appUser.id }, orderBy: { createdAt: "desc" }, take: 5 }),
-    listBackfillJobsForUser(appUser.id, 8),
+    listBackfillJobsForUser(appUser.id, 120),
   ]);
   const activities = activitiesRaw as ActivityRow[];
   const highlights = highlightsRaw as ManualHighlightRow[];

src/components/backfill-jobs-panel.tsx

@@ -4,9 +4,19 @@ import { useAppToast } from "@/components/providers";
 import { Button } from "@/components/ui/button";
 import { useBackfillActionMutation } from "@/lib/api/hooks";
 import { startSyncWatch } from "@/lib/sync-watch";
-import { AlertCircle, CheckCircle2, Clock3, Loader2, RefreshCw, Trash2, XCircle } from "lucide-react";
+import {
+  AlertCircle,
+  CheckCircle2,
+  ChevronDown,
+  ChevronRight,
+  Clock3,
+  Loader2,
+  RefreshCw,
+  Trash2,
+  XCircle,
+} from "lucide-react";
 import { useRouter } from "next/navigation";
-import { useState } from "react";
+import { useMemo, useState } from "react";
 
 export type BackfillJobRow = {
   id: string;
@@ -22,13 +32,76 @@ type Props = {
   jobs: BackfillJobRow[];
 };
 
+type BackfillGroup = {
+  key: string;
+  year: number;
+  provider: BackfillJobRow["provider"];
+  jobs: BackfillJobRow[];
+  queuedCount: number;
+  runningCount: number;
+  completedCount: number;
+  failedCount: number;
+  status: BackfillJobRow["status"];
+  latestError: string | null;
+};
+
 export function BackfillJobsPanel({ jobs }: Props) {
   const router = useRouter();
   const { pushToast } = useAppToast();
   const retryMutation = useBackfillActionMutation("retry");
   const deleteMutation = useBackfillActionMutation("delete");
   const cleanupMutation = useBackfillActionMutation("cleanup");
   const [busyAction, setBusyAction] = useState<string | null>(null);
+  const [expandedGroups, setExpandedGroups] = useState<Record<string, boolean>>({});
+
+  const groups = useMemo<BackfillGroup[]>(() => {
+    const map = new Map<string, BackfillGroup>();
+    for (const job of jobs) {
+      const key = `${job.year}:${job.provider ?? "ALL"}`;
+      const existing = map.get(key);
+      if (!existing) {
+        map.set(key, {
+          key,
+          year: job.year,
+          provider: job.provider,
+          jobs: [job],
+          queuedCount: job.status === "queued" ? 1 : 0,
+          runningCount: job.status === "running" ? 1 : 0,
+          completedCount: job.status === "completed" ? 1 : 0,
+          failedCount: job.status === "failed" ? 1 : 0,
+          status: job.status,
+          latestError: job.status === "failed" ? job.errorMessage : null,
+        });
+        continue;
+      }
+
+      existing.jobs.push(job);
+      if (job.status === "queued") existing.queuedCount += 1;
+      if (job.status === "running") existing.runningCount += 1;
+      if (job.status === "completed") existing.completedCount += 1;
+      if (job.status === "failed") {
+        existing.failedCount += 1;
+        if (!existing.latestError && job.errorMessage) existing.latestError = job.errorMessage;
+      }
+    }
+
+    const list = Array.from(map.values()).map((group) => {
+      const sortedJobs = [...group.jobs].sort((a, b) => {
+        const rank = { running: 0, queued: 1, failed: 2, completed: 3 } as const;
+        return rank[a.status] - rank[b.status];
+      });
+      let status: BackfillJobRow["status"] = "completed";
+      if (group.runningCount > 0) status = "running";
+      else if (group.queuedCount > 0) status = "queued";
+      else if (group.failedCount > 0) status = "failed";
+      return { ...group, jobs: sortedJobs, status };
+    });
+
+    return list.sort((a, b) => {
+      if (a.year !== b.year) return b.year - a.year;
+      return formatProvider(a.provider).localeCompare(formatProvider(b.provider));
+    });
+  }, [jobs]);
 
   async function postForm(action: "retry" | "delete" | "cleanup", jobId?: string) {
     const actionKey = `${action}:${jobId ?? "all"}`;
@@ -53,9 +126,40 @@ export function BackfillJobsPanel({ jobs }: Props) {
     router.refresh();
   }
 
+  async function runGroupAction(action: "retry-group" | "delete-group", group: BackfillGroup) {
+    const key = `${action}:${group.key}`;
+    setBusyAction(key);
+
+    try {
+      if (action === "retry-group") {
+        const failedJobs = group.jobs.filter((job) => job.status === "failed");
+        await Promise.all(failedJobs.map((job) => retryMutation.mutateAsync(job.id)));
+        if (failedJobs.length > 0) {
+          startSyncWatch();
+          pushToast(
+            { title: "Backfill retries queued", subtitle: `${failedJobs.length} failed chunk${failedJobs.length > 1 ? "s" : ""} requeued.` },
+            "info",
+          );
+        }
+      } else {
+        const deletableJobs = group.jobs.filter((job) => job.status !== "running");
+        await Promise.all(deletableJobs.map((job) => deleteMutation.mutateAsync(job.id)));
+        pushToast(
+          { title: "Backfill group cleaned", subtitle: `${deletableJobs.length} chunk${deletableJobs.length > 1 ? "s" : ""} removed.` },
+          "success",
+        );
+      }
+    } catch {
+      pushToast({ title: "Action failed", subtitle: "Please try again." }, "error");
+    } finally {
+      setBusyAction(null);
+      router.refresh();
+    }
+  }
+
   return (
     <div className="mt-4 space-y-2">
-      {jobs.length > 0 ? (
+      {groups.length > 0 ? (
         <div className="mb-2 flex justify-end">
           <Button
             size="sm"
@@ -64,61 +168,115 @@ export function BackfillJobsPanel({ jobs }: Props) {
             disabled={busyAction === "cleanup:all"}
             onClick={() => postForm("cleanup")}
           >
-            {busyAction === "cleanup:all" ? <span className="flex gap-1"><Loader2 className="size-4 animate-spin" /> Cleaning...</span> : <span className="flex gap-1"><Trash2 /> Clean completed</span>}
+            {busyAction === "cleanup:all" ? (
+              <span className="flex gap-1">
+                <Loader2 className="size-4 animate-spin" /> Cleaning...
+              </span>
+            ) : (
+              <span className="flex gap-1">
+                <Trash2 />
+                Clean completed
+              </span>
+            )}
           </Button>
         </div>
       ) : null}
 
-      {jobs.map((job) => (
-        <div key={job.id} className="flex flex-wrap items-center justify-between gap-3 rounded-md border border-border/60 bg-muted/20 px-3 py-2 text-sm">
-          <div className="flex items-center gap-2">
-            {renderBackfillStatusIcon(job.status)}
-            <span className="font-medium">{job.year}</span>
-            <span className="text-muted-foreground">• {formatProvider(job.provider)}</span>
-            <span className="text-muted-foreground">• {formatBackfillStatus(job.status)}</span>
-            <span className="text-muted-foreground">• attempt {job.attemptCount}</span>
-          </div>
-          <div className="flex items-center gap-2">
-            {job.status === "failed" && job.errorMessage ? (
-              <span className="max-w-[480px] truncate text-xs text-red-600" title={job.errorMessage}>
-                {job.errorMessage}
-              </span>
-            ) : null}
-            {job.status === "failed" ? (
-              <Button
-                size="sm"
-                variant="outline"
-                className="gap-1"
+      {groups.map((group) => {
+        const expanded = Boolean(expandedGroups[group.key]);
+        const canRetryFailed = group.failedCount > 0;
+        const deletableCount = group.jobs.filter((job) => job.status !== "running").length;
+
+        return (
+          <div key={group.key} className="rounded-md border border-border/60 bg-muted/20">
+            <div className="flex flex-wrap items-center justify-between gap-3 px-3 py-2 text-sm">
+              <button
                 type="button"
-                disabled={busyAction === `retry:${job.id}`}
-                onClick={() => postForm("retry", job.id)}
+                className="inline-flex items-center gap-2 text-left"
+                onClick={() => setExpandedGroups((prev) => ({ ...prev, [group.key]: !expanded }))}
               >
-                <RefreshCw className={busyAction === `retry:${job.id}` ? "size-4 animate-spin" : "size-4"} />
-                Retry
-              </Button>
+                {expanded ? <ChevronDown className="size-4 text-muted-foreground" /> : <ChevronRight className="size-4 text-muted-foreground" />}
+                {renderBackfillStatusIcon(group.status)}
+                <span className="font-medium">{group.year}</span>
+                <span className="text-muted-foreground">• {formatProvider(group.provider)}</span>
+                <span className="text-muted-foreground">• {group.completedCount}/{group.jobs.length} completed</span>
+                {group.runningCount > 0 ? <span className="text-blue-600">• {group.runningCount} running</span> : null}
+                {group.queuedCount > 0 ? <span className="text-amber-600">• {group.queuedCount} queued</span> : null}
+                {group.failedCount > 0 ? <span className="text-red-600">• {group.failedCount} failed</span> : null}
+              </button>
+              <div className="flex items-center gap-2">
+                {canRetryFailed ? (
+                  <Button
+                    size="sm"
+                    variant="outline"
+                    type="button"
+                    disabled={busyAction === `retry-group:${group.key}`}
+                    onClick={() => runGroupAction("retry-group", group)}
+                    className="gap-1"
+                  >
+                    <RefreshCw className={busyAction === `retry-group:${group.key}` ? "size-4 animate-spin" : "size-4"} />
+                    Retry failed
+                  </Button>
+                ) : null}
+                <Button
+                  size="icon"
+                  variant="destructive"
+                  type="button"
+                  disabled={deletableCount === 0 || busyAction === `delete-group:${group.key}`}
+                  onClick={() => runGroupAction("delete-group", group)}
+                >
+                  {busyAction === `delete-group:${group.key}` ? <Loader2 className="size-4 animate-spin" /> : <Trash2 className="size-4" />}
+                </Button>
+              </div>
+            </div>
+            {expanded ? (
+              <div className="space-y-2 border-t border-border/60 px-3 py-2">
+                {group.jobs.map((job) => (
+                  <div key={job.id} className="flex flex-wrap items-center justify-between gap-2 rounded-md border border-border/50 bg-background/50 px-2 py-1.5 text-xs">
+                    <div className="flex items-center gap-2">
+                      {renderBackfillStatusIcon(job.status)}
+                      <span className="text-muted-foreground">{formatBackfillStatus(job.status)}</span>
+                      <span className="text-muted-foreground">• attempt {job.attemptCount}</span>
+                      {job.status === "failed" && job.errorMessage ? (
+                        <span className="max-w-[480px] truncate text-red-600" title={job.errorMessage}>
+                          • {job.errorMessage}
+                        </span>
+                      ) : null}
+                    </div>
+                    <div className="flex items-center gap-2">
+                      {job.status === "failed" ? (
+                        <Button
+                          size="sm"
+                          variant="outline"
+                          type="button"
+                          className="h-7 gap-1 px-2 text-xs"
+                          disabled={busyAction === `retry:${job.id}`}
+                          onClick={() => postForm("retry", job.id)}
+                        >
+                          <RefreshCw className={busyAction === `retry:${job.id}` ? "size-3 animate-spin" : "size-3"} />
+                          Retry
+                        </Button>
+                      ) : null}
+                      <Button
+                        size="icon"
+                        variant="destructive"
+                        className="h-7 w-7"
+                        type="button"
+                        disabled={job.status === "running" || busyAction === `delete:${job.id}`}
+                        onClick={() => postForm("delete", job.id)}
+                      >
+                        {busyAction === `delete:${job.id}` ? <Loader2 className="size-3 animate-spin" /> : <Trash2 className="size-3" />}
+                      </Button>
+                    </div>
+                  </div>
+                ))}
+                {group.latestError ? <p className="text-xs text-red-600">{group.latestError}</p> : null}
+              </div>
             ) : null}
-            <Button
-              size="icon"
-              variant="destructive"
-              className="gap-1"
-              type="button"
-              disabled={busyAction === `delete:${job.id}`}
-              onClick={() => postForm("delete", job.id)}
-            >
-              {busyAction === `delete:${job.id}` ? (
-                <>
-                  <Loader2 className="size-4 animate-spin" />
-                </>
-              ) : (
-                <>
-                  <Trash2 className="size-4" />
-                </>
-              )}
-            </Button>
           </div>
-        </div>
-      ))}
-      {jobs.length === 0 ? <p className="text-sm text-muted-foreground">No historical backfill jobs yet.</p> : null}
+        );
+      })}
+      {groups.length === 0 ? <p className="text-sm text-muted-foreground">No historical backfill jobs yet.</p> : null}
     </div>
   );
 }

src/components/dashboard-live-updater.tsx

@@ -25,6 +25,8 @@ export function DashboardLiveUpdater() {
   const latestEventKey = useRef<string | null>(null);
   const hasAnnouncedStart = useRef(false);
   const latestTerminalRef = useRef<string | null>(null);
+  const latestRefreshedTerminalRef = useRef<string | null>(null);
+  const NO_ACTIVITY_TIMEOUT_MS = 3 * 60 * 1000;
 
   useEffect(() => {
     const source = new EventSource("/api/sync/events");
@@ -82,6 +84,11 @@ export function DashboardLiveUpdater() {
         const status = (await response.json()) as SyncStatusPayload;
         const watchStartedAt = getSyncWatchStartedAt();
         const activeJobCount = Number(status.activeJobCount ?? 0);
+        const terminalTime = status.latestJobFinishedAt ? Date.parse(status.latestJobFinishedAt) : NaN;
+        const terminalKey = status.latestJobFinishedAt ? `${status.latestJobStatus ?? "UNKNOWN"}:${status.latestJobFinishedAt}` : null;
+        const isFromCurrentWatch =
+          Number.isFinite(terminalTime) && watchStartedAt !== null ? terminalTime >= watchStartedAt : false;
+        const hasTerminalForCurrentWatch = Boolean(terminalKey && isFromCurrentWatch);
 
         if (activeJobCount > 0 && !hasAnnouncedStart.current) {
           hasAnnouncedStart.current = true;
@@ -95,12 +102,14 @@ export function DashboardLiveUpdater() {
           return;
         }
 
-        if (activeJobCount === 0 && hasAnnouncedStart.current) {
-          const terminalTime = status.latestJobFinishedAt ? Date.parse(status.latestJobFinishedAt) : NaN;
-          const terminalKey = status.latestJobFinishedAt ? `${status.latestJobStatus ?? "UNKNOWN"}:${status.latestJobFinishedAt}` : null;
-          const isFromCurrentWatch =
-            Number.isFinite(terminalTime) && watchStartedAt !== null ? terminalTime >= watchStartedAt : false;
+        // Incremental refresh: when a new chunk/job reaches terminal state for the current
+        // watch window, refresh dashboard so newly aggregated days appear immediately.
+        if (hasTerminalForCurrentWatch && terminalKey && latestRefreshedTerminalRef.current !== terminalKey) {
+          latestRefreshedTerminalRef.current = terminalKey;
+          router.refresh();
+        }
 
+        if (activeJobCount === 0 && hasTerminalForCurrentWatch) {
           if (terminalKey && latestTerminalRef.current !== terminalKey && isFromCurrentWatch) {
             latestTerminalRef.current = terminalKey;
             if (status.latestJobStatus === "FAILED") {
@@ -124,7 +133,17 @@ export function DashboardLiveUpdater() {
 
           clearSyncWatch();
           hasAnnouncedStart.current = false;
+          latestRefreshedTerminalRef.current = null;
           router.refresh();
+          return;
+        }
+
+        // Safety valve: if no active jobs are observed for a long time and no terminal
+        // event can be associated to this watch window, stop polling loop.
+        if (activeJobCount === 0 && watchStartedAt && Date.now() - watchStartedAt > NO_ACTIVITY_TIMEOUT_MS) {
+          clearSyncWatch();
+          hasAnnouncedStart.current = false;
+          latestRefreshedTerminalRef.current = null;
         }
       } catch {
         // ignore polling errors