feat: upgraded terminal ui

Author: caiopizzol

apps/scraper/commoncrawl/warc.ts

@@ -18,6 +18,7 @@ export interface FetchOptions {
   maxRetries?: number;
   maxBackoffMs?: number;
   rateLimiter?: RateLimiter;
+  onError?: (status: number, url: string, message: string) => void;
 }
 
 class HttpError extends Error {
@@ -37,7 +38,7 @@ export async function fetchWarcRecord(
   record: CdxRecord,
   options: FetchOptions = {},
 ): Promise<WarcResult> {
-  const { timeoutMs = 45000, maxRetries = 10, maxBackoffMs = 60000, rateLimiter } = options;
+  const { timeoutMs = 45000, maxRetries = 10, maxBackoffMs = 60000, rateLimiter, onError } = options;
 
   const offset = parseInt(record.offset, 10);
   const length = parseInt(record.length, 10);
@@ -71,6 +72,8 @@ export async function fetchWarcRecord(
       // Handle rate limiting errors with retry
       if (response.status === 503 || response.status === 429) {
         rateLimiter?.reportError(response.status);
+        const statusText = response.status === 429 ? "Too Many Requests" : "Service Unavailable";
+        onError?.(response.status, record.url, `${response.status} ${statusText} - backing off`);
         if (attempt < maxRetries) {
           const baseDelay = Math.min(2 ** attempt * 1000, maxBackoffMs);
           const jitter = Math.random() * 0.3 * baseDelay; // 0-30% jitter
@@ -85,6 +88,7 @@ export async function fetchWarcRecord(
       }
 
       if (!response.ok && response.status !== 206) {
+        onError?.(response.status, record.url, `${response.status} ${response.statusText}`);
         throw new HttpError(
           response.status,
           `WARC fetch failed: ${response.status} ${response.statusText}`,

apps/scraper/scraper.ts

@@ -12,9 +12,10 @@ import {
   blank,
   clearLines,
   formatDuration,
+  formatProgress,
   header,
   keyValue,
-  progressBar,
+  logError,
   section,
   writeMultiLineProgress,
 } from "./ui";
@@ -28,10 +29,11 @@ interface ProcessContext {
   stats: { saved: number; skipped: number; failed: number };
   rateLimiter: RateLimiter;
   force?: boolean;
+  onError?: (status: number, url: string, message: string) => void;
 }
 
 async function processRecord(record: CdxRecord, ctx: ProcessContext) {
-  const { db, storage, config, crawlId, stats, rateLimiter, force } = ctx;
+  const { db, storage, config, crawlId, stats, rateLimiter, force, onError } = ctx;
 
   // Check if already processed (skip if --force)
   if (!force) {
@@ -48,6 +50,7 @@ async function processRecord(record: CdxRecord, ctx: ProcessContext) {
     result = await fetchWarcRecord(record, {
       timeoutMs: config.crawl.timeoutMs,
       rateLimiter,
+      onError,
     });
   } catch (err) {
     stats.failed++;
@@ -169,40 +172,44 @@ export async function scrape(
   // Track throughput
   let lastThroughputUpdate = Date.now();
   let docsAtLastUpdate = 0;
+  let currentDocsPerSec = 0;
 
   // Track line count for clearing
-  let prevLineCount = 1;
+  let prevLineCount = 2;
+
+  // Error logging for verbose mode
+  const onError = verbose
+    ? (_status: number, url: string, message: string) => {
+        // Clear progress lines, log error, then redraw progress
+        clearLines(prevLineCount);
+        logError(`${message} - ${url}`);
+        prevLineCount = 0; // Reset so next update draws fresh
+      }
+    : undefined;
 
   // Progress update function
   const updateProgress = () => {
-    const lines: string[] = [];
-
     // Calculate docs/sec
     const now = Date.now();
     const elapsed = (now - lastThroughputUpdate) / 1000;
-    let docsPerSec = 0;
     if (elapsed >= 1) {
-      docsPerSec = (stats.saved - docsAtLastUpdate) / elapsed;
+      currentDocsPerSec = (stats.saved - docsAtLastUpdate) / elapsed;
       lastThroughputUpdate = now;
       docsAtLastUpdate = stats.saved;
     }
 
     const { errorCount } = rateLimiter.getStats();
 
-    const extras: string[] = [];
-    if (docsPerSec > 0) extras.push(`${docsPerSec.toFixed(1)}/s`);
-    if (stats.skipped > 0) extras.push(`${stats.skipped} dup`);
-    if (stats.failed > 0) extras.push(`${stats.failed} fail`);
-    if (errorCount > 0) extras.push(`${errorCount} retried`);
-    const extrasText = extras.length > 0 ? ` (${extras.join(" · ")})` : "";
-
-    if (batchSize === Infinity) {
-      lines.push(`  WARC: ${stats.saved} saved${extrasText}`);
-    } else {
-      const savedDisplay = Math.min(stats.saved, batchSize);
-      const warcBar = progressBar(savedDisplay, batchSize);
-      lines.push(`  WARC: ${warcBar} ${savedDisplay}/${batchSize} saved${extrasText}`);
-    }
+    const lines = formatProgress({
+      saved: Math.min(stats.saved, batchSize),
+      total: batchSize,
+      docsPerSec: currentDocsPerSec,
+      currentRps: rateLimiter.getCurrentRps(),
+      skipped: stats.skipped,
+      failed: stats.failed,
+      retried: errorCount,
+      elapsedMs: Date.now() - startTime,
+    });
 
     prevLineCount = writeMultiLineProgress(lines, prevLineCount);
   };
@@ -229,8 +236,9 @@ export async function scrape(
         config,
         crawlId,
         stats,
-        rateLimiter: rateLimiter,
+        rateLimiter,
         force,
+        onError,
       });
       updateProgress();
     }).finally(() => tasks.delete(task));

apps/scraper/ui.ts

@@ -52,3 +52,47 @@ export function formatDuration(ms: number): string {
   }
   return `${seconds}s`;
 }
+
+export interface ProgressStats {
+  saved: number;
+  total: number;
+  docsPerSec: number;
+  currentRps: number;
+  skipped: number;
+  failed: number;
+  retried: number;
+  elapsedMs: number;
+}
+
+export function formatProgress(stats: ProgressStats): string[] {
+  const { saved, total, docsPerSec, currentRps, skipped, failed, retried, elapsedMs } = stats;
+
+  const lines: string[] = [];
+
+  // Line 1: Progress bar with count and percentage
+  if (total === Infinity) {
+    lines.push(`━━━━━━━━━━━━━━━━━━━━ ${saved} saved`);
+  } else {
+    const bar = progressBar(saved, total);
+    const pct = total > 0 ? ((saved / total) * 100).toFixed(1) : "0.0";
+    lines.push(`${bar} ${saved}/${total} (${pct}%)`);
+  }
+
+  // Line 2: Metrics
+  const metrics: string[] = [];
+  metrics.push(`${docsPerSec.toFixed(1)}/s @ ${currentRps} RPS`);
+  if (skipped > 0) metrics.push(`${skipped} dup`);
+  if (failed > 0) metrics.push(`${failed} fail`);
+  if (retried > 0) metrics.push(`${retried} retried`);
+  metrics.push(formatDuration(elapsedMs));
+
+  lines.push(metrics.join(" · "));
+
+  return lines;
+}
+
+export function logError(message: string) {
+  const now = new Date();
+  const time = now.toLocaleTimeString("en-US", { hour12: false });
+  console.log(`[${time}] ${message}`);
+}