refac

Author: tjbck

backend/open_webui/retrieval/vector/dbs/pgvector.py

@@ -34,6 +34,7 @@
     SearchResult,
     GetResult,
 )
+from open_webui.utils.misc import sanitize_text_for_db
 from open_webui.config import (
     PGVECTOR_DB_URL,
     PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH,
@@ -289,7 +290,9 @@ def insert(self, collection_name: str, items: List[VectorItem]) -> None:
                     vector = self.adjust_vector_length(item['vector'])
                     # Use raw SQL for BYTEA/pgcrypto
                     # Ensure metadata is converted to its JSON text representation
-                    json_metadata = json.dumps(item['metadata'])
+                    # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
+                    json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
+                    item_text = sanitize_text_for_db(item['text'])
                     self.session.execute(
                         text("""
                             INSERT INTO document_chunk
@@ -305,7 +308,7 @@ def insert(self, collection_name: str, items: List[VectorItem]) -> None:
                             'id': item['id'],
                             'vector': vector,
                             'collection_name': collection_name,
-                            'text': item['text'],
+                            'text': item_text,
                             'metadata_text': json_metadata,
                             'key': PGVECTOR_PGCRYPTO_KEY,
                         },
@@ -338,7 +341,9 @@ def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
             if PGVECTOR_PGCRYPTO:
                 for item in items:
                     vector = self.adjust_vector_length(item['vector'])
-                    json_metadata = json.dumps(item['metadata'])
+                    # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store
+                    json_metadata = sanitize_text_for_db(json.dumps(item['metadata']))
+                    item_text = sanitize_text_for_db(item['text'])
                     self.session.execute(
                         text("""
                             INSERT INTO document_chunk
@@ -358,7 +363,7 @@ def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
                             'id': item['id'],
                             'vector': vector,
                             'collection_name': collection_name,
-                            'text': item['text'],
+                            'text': item_text,
                             'metadata_text': json_metadata,
                             'key': PGVECTOR_PGCRYPTO_KEY,
                         },

backend/open_webui/retrieval/vector/utils.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 
+from open_webui.utils.misc import sanitize_text_for_db
+
 KEYS_TO_EXCLUDE = ['content', 'pages', 'tables', 'paragraphs', 'sections', 'figures']
 
 
@@ -12,15 +14,17 @@ def filter_metadata(metadata: dict[str, any]) -> dict[str, any]:
 def process_metadata(
     metadata: dict[str, any],
 ) -> dict[str, any]:
-    # Removes large fields and converts non-serializable types (datetime, list, dict) to strings.
+    # Removes large fields, converts non-serializable types (datetime, list, dict) to strings,
+    # and sanitizes strings for database storage (strips null bytes and invalid surrogates).
     result = {}
     for key, value in metadata.items():
         # Skip large fields
         if key in KEYS_TO_EXCLUDE:
             continue
         # Convert non-serializable fields to strings
         if isinstance(value, (datetime, list, dict)):
-            result[key] = str(value)
+            result[key] = sanitize_text_for_db(str(value))
         else:
-            result[key] = value
+            result[key] = sanitize_text_for_db(value)
     return result
+