feat: lang detection

Author: caiopizzol

.env.example

@@ -23,7 +23,7 @@ EXTRACT_WORKERS=4                 # Parallel worker processes
 # Embedder settings
 EMBED_INPUT_PREFIX=extracted      # Input directory prefix (extracted text)
 EMBED_BATCH_SIZE=100              # Documents per batch
-EMBED_CONCURRENCY=20              # Parallel API requests (Tier 2: 5K RPM, adjust per tier)
+EMBED_CONCURRENCY=20              # Parallel API requests
 GOOGLE_API_KEY=                   # Required - get from https://aistudio.google.com/apikey
 
 # Cloudflare R2 (optional - for cloud storage)

db/migrations/002_add_language_columns.sql

@@ -0,0 +1,9 @@
+-- Migration: Add language detection columns
+-- Run this on existing databases to add language support
+
+-- Add language columns to extraction metadata
+ALTER TABLE documents ADD COLUMN IF NOT EXISTS language VARCHAR(10);
+ALTER TABLE documents ADD COLUMN IF NOT EXISTS language_confidence REAL;
+
+-- Create index for language filtering
+CREATE INDEX IF NOT EXISTS idx_documents_language ON documents(language) WHERE language IS NOT NULL;

db/schema.sql

@@ -22,6 +22,8 @@ CREATE TABLE IF NOT EXISTS documents (
     char_count INTEGER,
     table_count INTEGER,
     image_count INTEGER,
+    language VARCHAR(10),
+    language_confidence REAL,
     extraction_error TEXT,
 
     -- Embedding data
@@ -47,6 +49,7 @@ CREATE INDEX IF NOT EXISTS idx_documents_source_url ON documents(source_url);
 CREATE INDEX IF NOT EXISTS idx_documents_extracted ON documents(extracted_at) WHERE extracted_at IS NOT NULL;
 CREATE INDEX IF NOT EXISTS idx_documents_embedded ON documents(embedded_at) WHERE embedded_at IS NOT NULL;
 CREATE INDEX IF NOT EXISTS idx_documents_cluster ON documents(cluster_id) WHERE cluster_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_documents_language ON documents(language) WHERE language IS NOT NULL;
 
 -- Vector similarity search index (IVFFlat for approximate nearest neighbor)
 -- Note: Run this AFTER populating embeddings for better index quality

packages/extractor/processor.ts

@@ -84,6 +84,8 @@ class PersistentExtractor {
     charCount?: number;
     tableCount?: number;
     imageCount?: number;
+    language?: string;
+    languageConfidence?: number;
     extraction?: any;
     error?: string;
   }> {
@@ -268,6 +270,8 @@ async function processBatch(
       char_count: result.charCount!,
       table_count: result.tableCount!,
       image_count: result.imageCount!,
+      language: result.language || "unknown",
+      language_confidence: result.languageConfidence || 0,
       extracted_at: new Date().toISOString(),
     });
 

packages/extractor/python/extract_server.py

@@ -21,6 +21,23 @@
 from docling.datamodel.base_models import InputFormat
 from docling_core.types.doc.labels import DocItemLabel
 
+import langid
+
+
+def detect_language(text: str, min_chars: int = 50) -> tuple[str, float]:
+    """Detect language using langid. Returns (lang_code, confidence)."""
+    if not text or len(text) < min_chars:
+        return "unknown", 0.0
+
+    try:
+        lang, score = langid.classify(text[:2000])
+        # Normalize confidence: langid scores are negative log-probs, typically -500 to -3000
+        # Map to 0-1 where closer to 0 = higher confidence
+        confidence = max(0.0, min(1.0, 1.0 + score / 3000))
+        return lang, confidence
+    except Exception:
+        return "unknown", 0.0
+
 
 @contextlib.contextmanager
 def suppress_stderr():
@@ -85,6 +102,9 @@ def extract(converter: DocumentConverter, file_path: str) -> dict:
     # Use smart extraction to avoid table padding bloat
     text = smart_extract_text(result.document)
 
+    # Detect language
+    lang, lang_confidence = detect_language(text)
+
     # Get full structured extraction (stripped of image data)
     extraction = result.document.export_to_dict()
     extraction = strip_image_data(extraction)
@@ -95,6 +115,8 @@ def extract(converter: DocumentConverter, file_path: str) -> dict:
         "charCount": len(text),
         "tableCount": len(extraction.get("tables", [])),
         "imageCount": len(extraction.get("pictures", [])),
+        "language": lang,
+        "languageConfidence": lang_confidence,
         "extraction": extraction,
     }
 

packages/extractor/python/pyproject.toml

@@ -2,4 +2,7 @@
 name = "text-extractor-python"
 version = "0.1.0"
 requires-python = ">=3.10"
-dependencies = ["docling>=2.0.0"]
+dependencies = [
+    "docling>=2.0.0",
+    "langid>=1.1.6",
+]

packages/extractor/types.ts

@@ -11,6 +11,8 @@ export interface ExtractedDocument {
   charCount: number;
   tableCount: number;
   imageCount: number;
+  language: string;
+  languageConfidence: number;
   extraction: Record<string, unknown>;
   extractedAt: string;
 }

packages/shared/db.ts

@@ -21,6 +21,8 @@ export interface DocumentRecord {
   char_count: number | null;
   table_count: number | null;
   image_count: number | null;
+  language: string | null;
+  language_confidence: number | null;
   extraction_error: string | null;
 
   // Embedding data
@@ -40,6 +42,8 @@ export interface ExtractionData {
   char_count: number;
   table_count: number;
   image_count: number;
+  language: string;
+  language_confidence: number;
   extracted_at?: string;
   extraction_error?: string;
 }
@@ -200,6 +204,8 @@ export async function createDb(databaseUrl: string): Promise<DbClient> {
           char_count = ${data.char_count},
           table_count = ${data.table_count},
           image_count = ${data.image_count},
+          language = ${data.language},
+          language_confidence = ${data.language_confidence},
           extraction_error = NULL
         WHERE id = ${data.id}
       `;