feat(server): update full_content scrapping

Author: Antoine0703

server/__pycache__/config.cpython-312.pyc

@@ -42,6 +42,8 @@ def setup_database(self):
                 source_site TEXT NOT NULL,
                 title TEXT NOT NULL,
                 description TEXT,
+                full_content TEXT,
+                content_hash TEXT UNIQUE,
                 author_info TEXT,
                 keywords TEXT,
                 content_url TEXT NOT NULL,
@@ -74,6 +76,9 @@ def setup_database(self):
             )
             """)
 
+            cur.execute("CREATE INDEX IF NOT EXISTS idx_content_hash ON articles(content_hash)")
+            cur.execute("CREATE INDEX IF NOT EXISTS idx_source_site ON articles(source_site)")
+            
             conn.commit()
 
     def save_article(self, item: Dict, conn: Optional[sqlite3.Connection] = None) -> bool:
@@ -87,6 +92,8 @@ def save_article(self, item: Dict, conn: Optional[sqlite3.Connection] = None) ->
         Returns:
             True if inserted, False if already exists
         """
+        import hashlib
+        
         should_close = False
         if conn is None:
             conn = sqlite3.connect(self.db_path)
@@ -95,15 +102,20 @@ def save_article(self, item: Dict, conn: Optional[sqlite3.Connection] = None) ->
         try:
             cur = conn.cursor()
 
+            full_content = item.get("full_content", item.get("description", ""))
+            content_hash = hashlib.sha256(full_content.encode()).hexdigest()
+            
             cur.execute("""
             INSERT OR IGNORE INTO articles
-            (id, source_site, title, description, author_info, keywords, content_url, published_date, item_type, created_at)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            (id, source_site, title, description, full_content, content_hash, author_info, keywords, content_url, published_date, item_type, created_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             """, (
                 item["id"],
                 item["source_site"],
                 item["title"],
                 item.get("description", ""),
+                full_content,
+                content_hash,
                 item.get("author_info", ""),
                 item.get("keywords", ""),
                 item["content_url"],
@@ -137,12 +149,43 @@ def save_articles_batch(self, items: List[Dict]) -> int:
             return count
 
     def article_exists(self, article_id: str) -> bool:
-        """Check if article already exists."""
+        """Check if article already exists by ID."""
         with self.get_connection() as conn:
             cur = conn.cursor()
             cur.execute("SELECT 1 FROM articles WHERE id = ?", (article_id,))
             return cur.fetchone() is not None
 
+    def article_exists_by_hash(self, content_hash: str) -> bool:
+        """
+        Check if article already exists by content hash.
+        
+        Args:
+            content_hash: SHA256 hash of article content
+            
+        Returns:
+            True if article with same content exists
+        """
+        with self.get_connection() as conn:
+            cur = conn.cursor()
+            cur.execute("SELECT 1 FROM articles WHERE content_hash = ?", (content_hash,))
+            return cur.fetchone() is not None
+    
+    def get_article_by_hash(self, content_hash: str) -> Optional[Dict]:
+        """
+        Get article by content hash.
+        
+        Args:
+            content_hash: SHA256 hash of article content
+            
+        Returns:
+            Article dict if found, None otherwise
+        """
+        with self.get_connection() as conn:
+            cur = conn.cursor()
+            cur.execute("SELECT * FROM articles WHERE content_hash = ?", (content_hash,))
+            row = cur.fetchone()
+            return dict(row) if row else None
+    
     def save_embedding(self, article_id: str, embedding: bytes, model: str = "default") -> bool:
         """
         Save article embedding.

server/__pycache__/database.cpython-312.pyc

@@ -111,15 +111,21 @@ def embed_article(self, article: dict) -> bytes:
         """
         Generate embedding for complete article.
         
+        Uses full_content if available, otherwise combines title and description.
+        
         Args:
-            article: Dict with title and description
+            article: Dict with title, description, and optional full_content
             
         Returns:
             Serialized embedding in bytes
         """
-        title = article.get("title", "")
-        description = article.get("description", "")
-        text = f"{title}\n{description}"
+        full_content = article.get("full_content")
+        if full_content:
+            text = full_content
+        else:
+            title = article.get("title", "")
+            description = article.get("description", "")
+            text = f"{title}\n{description}"
 
         return self.embed_text(text)
 

server/__pycache__/embeddings.cpython-312.pyc

@@ -105,12 +105,21 @@ def _process_articles(self, articles: List[Dict]) -> int:
         Returns:
             Number of new articles processed
         """
+        import hashlib
+        
         new_count = 0
 
         for article in articles:
             if self.db_manager.article_exists(article["id"]):
                 continue
 
+            full_content = article.get("full_content", article.get("description", ""))
+            content_hash = hashlib.sha256(full_content.encode()).hexdigest()
+            
+            if self.db_manager.article_exists_by_hash(content_hash):
+                logger.debug(f"Article {article['id']} is duplicate (same content hash)")
+                continue
+            
             if self.db_manager.save_article(article):
                 new_count += 1
 

server/__pycache__/main.cpython-312.pyc

@@ -2,3 +2,5 @@ feedparser==6.0.10
 requests==2.31.0
 arxiv==1.4.8
 numpy==1.26.4
+beautifulsoup4==4.12.2
+PyPDF2==3.0.1