feat: skip failed exec

Author: caiopizzol

packages/extractor/processor.ts

@@ -56,7 +56,7 @@ export async function processDirectory(
   console.log(`  Output: ${outputPrefix}/{hash}.txt`);
 }
 
-const EXTRACTION_TIMEOUT_MS = 120_000; // 2 minutes per document
+const EXTRACTION_TIMEOUT_MS = 30_000; // 30 seconds per document
 
 async function extractWithPython(
   doc: DocumentRecord,

packages/scraper/scraper.ts

@@ -164,8 +164,10 @@ export async function scrape(options: ScrapeOptions) {
   // Initialize database
   const db = await createDb(config.database.url);
 
-  // Pre-load successful URLs for fast duplicate checking
+  // Pre-load processed URLs for fast duplicate checking (includes both uploaded and failed)
   const uploadedUrls = force ? new Set<string>() : await db.getUploadedUrls();
+  const failedUrls = force ? new Set<string>() : await db.getFailedUrls();
+  const processedUrls = new Set([...uploadedUrls, ...failedUrls]);
 
   // Aggregate stats across all crawls
   const totalStats = { saved: 0, skipped: 0, failed: 0 };
@@ -247,7 +249,7 @@ export async function scrape(options: ScrapeOptions) {
 
       const task = downloadLimit(async () => {
         // Check duplicates BEFORE rate limiting (instant skip)
-        if (!force && uploadedUrls.has(record.url)) {
+        if (!force && processedUrls.has(record.url)) {
           stats.skipped++;
           updateProgress();
           return;

packages/shared/db.ts

@@ -64,6 +64,7 @@ export interface DbClient {
   getDocument(id: string): Promise<DocumentRecord | null>;
   getDocumentByUrl(url: string): Promise<DocumentRecord | null>;
   getUploadedUrls(): Promise<Set<string>>;
+  getFailedUrls(): Promise<Set<string>>;
   getPendingDocuments(limit: number): Promise<DocumentRecord[]>;
   getDocumentsByStatus(status: DocumentStatus, limit?: number): Promise<DocumentRecord[]>;
   getStats(): Promise<{ status: string; count: number }[]>;
@@ -157,6 +158,13 @@ export async function createDb(databaseUrl: string): Promise<DbClient> {
       return new Set(rows.map((r) => r.source_url));
     },
 
+    async getFailedUrls() {
+      const rows = await sql<{ source_url: string }[]>`
+        SELECT source_url FROM documents WHERE status = 'failed'
+      `;
+      return new Set(rows.map((r) => r.source_url));
+    },
+
     async getPendingDocuments(limit: number) {
       return sql<DocumentRecord[]>`
         SELECT * FROM documents WHERE status = 'pending' LIMIT ${limit}
@@ -207,11 +215,12 @@ export async function createDb(databaseUrl: string): Promise<DbClient> {
     },
 
     async getUnextractedDocuments(limit: number) {
-      // Get docs that haven't been extracted yet (includes failed ones for retry)
+      // Get docs that haven't been extracted yet (excludes previously failed ones)
       return sql<DocumentRecord[]>`
         SELECT * FROM documents
         WHERE status = 'uploaded'
           AND extracted_at IS NULL
+          AND extraction_error IS NULL
         ORDER BY uploaded_at ASC
         LIMIT ${limit}
       `;