feat: save raw extraction

Author: caiopizzol

packages/extractor/processor.ts

@@ -53,7 +53,7 @@ export async function processDirectory(
   }
 
   console.log("\nExtraction complete!");
-  console.log(`  Output: ${outputPrefix}/{hash}.txt`);
+  console.log(`  Output: ${outputPrefix}/{hash}.txt, ${outputPrefix}/{hash}.json`);
 }
 
 const EXTRACTION_TIMEOUT_MS = 30_000; // 30 seconds per document
@@ -106,6 +106,7 @@ async function extractWithPython(
     charCount: result.charCount,
     tableCount: result.tableCount,
     imageCount: result.imageCount,
+    extraction: result.extraction,
     extractedAt: new Date().toISOString(),
   };
 }
@@ -173,6 +174,12 @@ async function processBatch(
         // Write text file to storage
         await storage.write(`${outputPrefix}/${doc.id}.txt`, extracted.text);
 
+        // Write extraction JSON to storage
+        await storage.write(
+          `${outputPrefix}/${doc.id}.json`,
+          JSON.stringify(extracted.extraction)
+        );
+
         // Update database with extraction metadata
         await db.updateExtraction({
           id: doc.id,

packages/extractor/python/extract.py

@@ -15,15 +15,16 @@ def extract(file_path: str) -> dict:
     # Export as markdown for text extraction
     text = result.document.export_to_markdown()
 
-    # Get structured data for counts
-    doc_dict = result.document.export_to_dict()
+    # Get full structured extraction (raw, no stripping)
+    extraction = result.document.export_to_dict()
 
     return {
         "text": text,
         "wordCount": len(text.split()),
         "charCount": len(text),
-        "tableCount": len(doc_dict.get("tables", [])),
-        "imageCount": len(doc_dict.get("pictures", [])),
+        "tableCount": len(extraction.get("tables", [])),
+        "imageCount": len(extraction.get("pictures", [])),
+        "extraction": extraction,
     }
 
 

packages/extractor/types.ts

@@ -11,6 +11,7 @@ export interface ExtractedDocument {
   charCount: number;
   tableCount: number;
   imageCount: number;
+  extraction: Record<string, unknown>;
   extractedAt: string;
 }