fix(scrapping): fix embedding model

Author: Antoine0703

server/.env.example

@@ -0,0 +1,5 @@
+# OpenAI API Configuration
+OPENAI_API_KEY=your_openai_api_key_here
+
+# Optional: GitHub Token for higher rate limits
+GITHUB_TOKEN=your_github_token_here

server/README.md

@@ -23,9 +23,6 @@ class ServerConfig:
 
     watch_interval_seconds: int = 300
 
-    use_dummy_embeddings: bool = True
-    embedding_model: str = "all-MiniLM-L6-v2"
-    
     log_level: str = "INFO"
 
     scrapers: Dict[str, ScraperConfig] = None
@@ -66,13 +63,11 @@ def from_file(cls, filepath: str) -> "ServerConfig":
 
 DEV_CONFIG = ServerConfig(
     db_path="veille_technique_dev.db",
-    use_dummy_embeddings=True,
     watch_interval_seconds=60,
 )
 
 PROD_CONFIG = ServerConfig(
     db_path="veille_technique.db",
-    use_dummy_embeddings=False,
     watch_interval_seconds=600,
     log_level="WARNING",
 )

server/config.py

@@ -4,6 +4,8 @@
 from typing import List, Optional
 from abc import ABC, abstractmethod
 import numpy as np
+import os
+from pathlib import Path
 
 
 class EmbeddingProvider(ABC):
@@ -82,6 +84,75 @@ def get_name(self) -> str:
         return f"sentence-transformers-{self.model_name}"
 
 
+class OpenAIEmbeddingProvider(EmbeddingProvider):
+    """Embedding provider using OpenAI API."""
+    
+    def __init__(self, model: str = "text-embedding-3-small", api_key: Optional[str] = None):
+        """
+        Initialize OpenAI embedding provider.
+        
+        Args:
+            model: OpenAI embedding model to use (text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002)
+            api_key: OpenAI API key (defaults to .env file or OPENAI_API_KEY env var)
+        """
+        try:
+            from openai import OpenAI
+        except ImportError:
+            raise ImportError(
+                "openai package is required. Install it with: "
+                "pip install openai"
+            )
+        
+        self.model = model
+        
+        if api_key:
+            self.api_key = api_key
+        else:
+            self.api_key = self._load_api_key_from_env()
+        
+        if not self.api_key:
+            raise ValueError(
+                "OpenAI API key is required. Add OPENAI_API_KEY to .env file "
+                "or set OPENAI_API_KEY environment variable."
+            )
+        
+        self.client = OpenAI(api_key=self.api_key)
+    
+    def _load_api_key_from_env(self) -> Optional[str]:
+        """Load API key from .env file or environment variable."""
+        api_key = os.getenv("OPENAI_API_KEY")
+        if api_key:
+            return api_key
+        
+        env_path = Path(__file__).parent / ".env"
+        if env_path.exists():
+            with open(env_path, 'r') as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith('OPENAI_API_KEY='):
+                        return line.split('=', 1)[1].strip().strip('"').strip("'")
+        
+        return None
+    
+    def embed(self, text: str) -> np.ndarray:
+        """Generate embedding with OpenAI API."""
+        max_chars = 30000
+        if len(text) > max_chars:
+            text = text[:max_chars]
+        
+        response = self.client.embeddings.create(
+            input=text,
+            model=self.model
+        )
+        
+        embedding = np.array(response.data[0].embedding, dtype=np.float32)
+        return embedding
+    
+    def get_name(self) -> str:
+        """Return provider name."""
+        return f"openai-{self.model}"
+
+
 class EmbeddingManager:
     """Manage embeddings for articles."""
 

server/embeddings.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """
 Usage examples for the watch server.
 """
@@ -16,7 +15,6 @@ def example_backfill():
 
     server = WatchServer(
         db_path="test_backfill.db",
-        use_dummy_embeddings=True,
         check_interval=300
     )
 
@@ -34,7 +32,6 @@ def example_watch_limited():
 
     server = WatchServer(
         db_path="test_watch.db",
-        use_dummy_embeddings=True,
         check_interval=10
     )
 
@@ -96,7 +93,6 @@ def example_custom_config():
     custom_config = ServerConfig(
         db_path="test_custom.db",
         watch_interval_seconds=120,
-        use_dummy_embeddings=True,
         scrapers={
             "arxiv": ScraperConfig(enabled=True, limit_latest=10, limit_all=30),
             "github": ScraperConfig(enabled=True, limit_latest=15, limit_all=50),
@@ -109,7 +105,6 @@ def example_custom_config():
     print(f"\n✓ Custom config created")
     print(f"  DB : {custom_config.db_path}")
     print(f"  Interval : {custom_config.watch_interval_seconds}s")
-    print(f"  Dummy embeddings : {custom_config.use_dummy_embeddings}")
 
     print(f"\n✓ Enabled scrapers :")
     for name, cfg in custom_config.scrapers.items():