feat: default latest crawl id

Author: caiopizzol

apps/scraper/__tests__/db.test.ts

@@ -38,7 +38,7 @@ describe.skipIf(!postgresAvailable)("db", () => {
     // Clean up test data from previous runs
     const { SQL } = await import("bun");
     const sql = new SQL({ url: DATABASE_URL });
-    await sql`DELETE FROM documents`;
+    await sql`TRUNCATE documents`;
     await sql.close();
 
     db = await createDb(DATABASE_URL);

apps/scraper/commoncrawl/index.ts

@@ -0,0 +1,12 @@
+const COLLINFO_URL = "https://index.commoncrawl.org/collinfo.json";
+
+interface CrawlInfo {
+  id: string;
+  name: string;
+}
+
+export async function getLatestCrawlId(): Promise<string> {
+  const res = await fetch(COLLINFO_URL);
+  const data = (await res.json()) as CrawlInfo[];
+  return data[0].id;
+}

apps/scraper/index.ts

@@ -15,7 +15,7 @@ Commands
 
 Options
   --batch <n>   Limit to n documents (default: all)
-  --crawl <id>  Common Crawl index ID (required, or set CRAWL_ID env var)
+  --crawl <id>  Common Crawl index ID (default: latest)
   --force       Re-process URLs already in database
   --verbose     Show detailed logs for debugging
 

apps/scraper/scraper.ts

@@ -1,5 +1,6 @@
 import pLimit from "p-limit";
 import { streamCdxFromR2, type CdxRecord } from "./commoncrawl/cdx-r2";
+import { getLatestCrawlId } from "./commoncrawl/index";
 import { fetchWarcRecord, type WarcResult } from "./commoncrawl/warc";
 import { type Config, hasCloudflareCredentials } from "./config";
 import { generateManifest } from "./manifest";
@@ -124,9 +125,9 @@ export async function scrape(
 
   header();
 
-  const crawlId = config.crawl.id;
+  let crawlId = config.crawl.id;
   if (!crawlId) {
-    throw new Error("CRAWL_ID must be specified");
+    crawlId = await getLatestCrawlId();
   }
 
   section("Configuration");