fixed transcript retreival bugs.

Author: cledantec

main_chat/chat_route.py

@@ -277,7 +277,7 @@ def _route_question(question: str) -> Dict[str, Any]:
         "3. If mode is 'rag' or 'hybrid':\n"
         "   - transcript_tags: array of 0-2 strings OR null (valid tags: safety, violence, youth, media, community, displacement, government, structural racism)\n"
         "   - policy_sources: array of strings OR null (valid: 'Boston Anti-Displacement Plan Analysis.txt', 'Boston Slow Streets Plan Analysis.txt', 'Imagine Boston 2030 Analysis.txt')\n"
-        "   - folder_categories: array of strings OR null (valid: newsletters, policy, transcripts)\n"
+        "   - folder_categories: array of strings OR null (valid: newsletters, policies, transcripts)\n"
         "   - k: integer between 3 and 10 (default 5, minimum 5 for event queries)\n"
         "4. For crime questions using 'hybrid' mode (Rule 1b or 1c): transcript_tags MUST include at least one of: 'safety' or 'violence'\n"
         "   For crime questions using 'sql' mode (Rule 1a): transcript_tags, policy_sources, and folder_categories MUST be null\n"
@@ -583,7 +583,6 @@ def _run_rag(question: str, plan: Dict[str, Any], conversation_history: Optional
     combined_chunks: List[str] = []
     combined_meta: List[Dict[str, Any]] = []
 
-    # Increase k for better source diversity
     retrieval_k = max(k * 3, 20)
 
     # ========================================================================
@@ -599,7 +598,7 @@ def _run_rag(question: str, plan: Dict[str, Any], conversation_history: Optional
         print(f"  ⚠ Transcript retrieval error: {e}")
 
     # ========================================================================
-    # POLICIES - retrieve from CLIENT_UPLOAD with folder_category filter
+    # POLICIES - retrieve from client_upload with folder_category filter
     # ========================================================================
     try:
         if sources:
@@ -618,11 +617,17 @@ def _run_rag(question: str, plan: Dict[str, Any], conversation_history: Optional
             folders_list = folders if isinstance(folders, list) else [folders]
 
             for folder in folders_list:
-                p_res = rag_retrieval.retrieve(question, k=retrieval_k, doc_type="client_upload", folder_category=folder)
-                p_chunks = p_res.get("chunks", [])
-                print(f"    Found {len(p_chunks)} chunks from folder: {folder}")
-                combined_chunks.extend(p_chunks)
-                combined_meta.extend(p_res.get("metadata", []))
+                # Map folder to correct doc_type
+                if folder == "transcripts":
+                    # Transcripts have doc_type="transcript" (already retrieved above)
+                    continue
+                else:
+                    # Other folders (policies, newsletters) are client_upload
+                    p_res = rag_retrieval.retrieve(question, k=retrieval_k, doc_type="client_upload", folder_category=folder)
+                    p_chunks = p_res.get("chunks", [])
+                    print(f"    Found {len(p_chunks)} chunks from folder: {folder}")
+                    combined_chunks.extend(p_chunks)
+                    combined_meta.extend(p_res.get("metadata", []))
         else:
             # No specific sources or folders - search all policies by default
             print("  📚 No specific policy sources, searching all policies")

main_chat/rag_pipeline/rag_retrieval.py

@@ -38,7 +38,6 @@ def load_vectordb():
 def retrieve(query, k=5, doc_type=None, tags=None, source=None, folder_category=None, min_score=None, vectordb=None):
     """
     Universal retrieval with flexible metadata filtering.
-    Added 'folder_category' parameter to filter by Google Drive subfolder.
     """
     # Defensive clamp: Chroma requires k >= 1
     try:
@@ -52,45 +51,45 @@ def retrieve(query, k=5, doc_type=None, tags=None, source=None, folder_category=
         vectordb = load_vectordb()
 
     # Build filter dictionary
-    filter_conditions = []
+    filter_dict = None
+
+    # Build list of filter conditions
+    filters = []
 
     # Doc type filter
-    doc_filter = None
-    if isinstance(doc_type, (list, tuple)):
-        doc_types = [dt for dt in doc_type if dt]
-        if len(doc_types) == 1:
-            doc_filter = {"doc_type": doc_types[0]}
-        elif len(doc_types) > 1:
-            doc_filter = {"$or": [{"doc_type": dt} for dt in doc_types]}
-    elif doc_type:
-        doc_filter = {"doc_type": doc_type}
-
-    if doc_filter:
-        filter_conditions.append(doc_filter)
+    if doc_type:
+        if isinstance(doc_type, (list, tuple)):
+            doc_types = [dt for dt in doc_type if dt]
+            if len(doc_types) == 1:
+                filters.append({"doc_type": doc_types[0]})
+            elif len(doc_types) > 1:
+                filters.append({"$or": [{"doc_type": dt} for dt in doc_types]})
+        else:
+            filters.append({"doc_type": doc_type})
 
     # Source filter
     if source:
-        filter_conditions.append({"source": source})
+        filters.append({"source": source})
 
-    # Folder category filter (NEW)
+    # Folder category filter
     if folder_category:
         if isinstance(folder_category, (list, tuple)):
             if len(folder_category) == 1:
-                filter_conditions.append({"folder_category": folder_category[0]})
+                filters.append({"folder_category": folder_category[0]})
             elif len(folder_category) > 1:
-                filter_conditions.append({"$or": [{"folder_category": f} for f in folder_category]})
+                filters.append({"$or": [{"folder_category": f} for f in folder_category]})
         else:
-            filter_conditions.append({"folder_category": folder_category})
+            filters.append({"folder_category": folder_category})
 
-    # Combine all conditions
-    filter_dict = None
-    if len(filter_conditions) == 1:
-        filter_dict = filter_conditions[0]
-    elif len(filter_conditions) > 1:
-        filter_dict = {"$and": filter_conditions}
+    # Combine filters
+    if len(filters) == 1:
+        filter_dict = filters[0]
+    elif len(filters) > 1:
+        filter_dict = {"$and": filters}
 
+    # Retrieve with or without min_score
     if min_score is not None:
-        results_with_scores = vectordb.similarity_search_with_score(query, k=k * 3 if tags else k, filter=filter_dict if filter_dict else None)
+        results_with_scores = vectordb.similarity_search_with_score(query, k=k * 3 if tags else k, filter=filter_dict)
 
         if tags:
             filtered_results = []
@@ -106,9 +105,14 @@ def retrieve(query, k=5, doc_type=None, tags=None, source=None, folder_category=
 
         filtered_results = [(doc, score) for doc, score in results_with_scores if score <= min_score]
 
-        return {"chunks": [doc.page_content for doc, _ in filtered_results[:k]], "metadata": [doc.metadata for doc, _ in filtered_results[:k]], "scores": [score for _, score in filtered_results[:k]], "query": query}
+        return {
+            "chunks": [doc.page_content for doc, _ in filtered_results[:k]],
+            "metadata": [doc.metadata for doc, _ in filtered_results[:k]],
+            "scores": [score for _, score in filtered_results[:k]],
+            "query": query,
+        }
     else:
-        results = vectordb.similarity_search(query, k=k * 3 if tags else k, filter=filter_dict if filter_dict else None)
+        results = vectordb.similarity_search(query, k=k * 3 if tags else k, filter=filter_dict)
 
         if tags:
             filtered_results = []
@@ -122,12 +126,17 @@ def retrieve(query, k=5, doc_type=None, tags=None, source=None, folder_category=
             if filtered_results:
                 results = filtered_results
 
-        return {"chunks": [doc.page_content for doc in results[:k]], "metadata": [doc.metadata for doc in results[:k]], "scores": None, "query": query}
+        return {
+            "chunks": [doc.page_content for doc in results[:k]],
+            "metadata": [doc.metadata for doc in results[:k]],
+            "scores": None,
+            "query": query,
+        }
 
 
 def retrieve_transcripts(query, tags=None, k=5):
     """Convenience function for transcript-only search."""
-    return retrieve(query, k=k, doc_type="transcripts", tags=tags)
+    return retrieve(query, k=k, doc_type="transcript", tags=tags)
 
 
 def retrieve_policies(query, k=5, source=None):