testing/debug tools

Author: cledantec

main_chat/tests/inspect_vector_db.py

@@ -10,7 +10,6 @@
 _PROJECT_ROOT = Path(__file__).resolve().parents[2]
 sys.path.insert(0, str(_PROJECT_ROOT))
 
-import config
 from main_chat.rag_pipeline.rag_retrieval import load_vectordb
 
 
@@ -56,6 +55,17 @@ def inspect_vectordb():
         for dtype, count in sorted(doc_types.items()):
             print(f"  - {dtype}: {count}")
 
+        print("\n" + "=" * 80)
+        print("CLIENT_UPLOAD METADATA CHECK")
+        print("=" * 80)
+
+        for meta in metadatas:
+            if meta.get("doc_type") == "client_upload" and meta.get("chunk_id") == 0:
+                print(f"\nCLIENT_UPLOAD document:")
+                print(f"  Source: {meta.get('source')}")
+                print(f"  Full metadata: {meta}")
+                print(f"  Has folder_category? {meta.get('folder_category')}")
+
         # Show ALL files/sources grouped by document type
         print("\n" + "=" * 80)
         print("📁 ALL FILES IN VECTOR DATABASE (by type)")
@@ -88,12 +98,12 @@ def inspect_vectordb():
         print("Testing Policy Retrieval")
         print("=" * 80)
 
-        test_query = "anti-displacement"
-        print(f"\nTest query: '{test_query}'")
-
         # Try retrieving policies
         from main_chat.rag_pipeline.rag_retrieval import retrieve_policies
 
+        test_query = "anti-displacement"
+        print(f"\nTest query: '{test_query}'")
+
         result = retrieve_policies(test_query, k=5)
         chunks = result.get("chunks", [])
         metadata = result.get("metadata", [])
@@ -105,10 +115,11 @@ def inspect_vectordb():
             for i, (chunk, meta) in enumerate(zip(chunks[:2], metadata[:2]), 1):
                 print(f"\n  Result {i}:")
                 print(f"  Source: {meta.get('source', 'unknown')}")
-                print(f"  Type: {meta.get('doc_type', 'unknown')}")
+                print(f"  Doc Type: {meta.get('doc_type', 'unknown')}")
+                print(f"  Folder Category: {meta.get('folder_category', 'none')}")
                 print(f"  Preview: {chunk[:200]}...")
         else:
-            print("✗ No chunks retrieved! This is the problem.")
+            print("✗ No chunks retrieved!")
             # Try a more specific source filter
             print("\nTrying with specific source filter...")
             result2 = retrieve_policies(test_query, k=5, source="Boston Anti-Displacement Plan Analysis.txt")

main_chat/tests/test_metadata.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+import sys
+
+# test_policy_retrieval.py
+
+_PROJECT_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(_PROJECT_ROOT))
+# check_metadata.py
+from main_chat.rag_pipeline.rag_retrieval import load_vectordb
+
+vectordb = load_vectordb()
+all_docs = vectordb.get(limit=10000)
+
+# Find CLIENT_UPLOAD docs and show their metadata
+print("=" * 80)
+print("CLIENT_UPLOAD DOCUMENTS - ACTUAL METADATA")
+print("=" * 80)
+
+for i, meta in enumerate(all_docs["metadatas"]):
+    doc_type = meta.get("doc_type", "")
+
+    # Check all possible variations
+    if "CLIENT" in str(doc_type).upper() or "UPLOAD" in str(doc_type).upper():
+        print(f"\nDocument {i+1}:")
+        print(f"  doc_type: '{doc_type}' (type: {type(doc_type).__name__})")
+        print(f"  folder_category: '{meta.get('folder_category')}' (exists: {('folder_category' in meta)})")
+        print(f"  source: '{meta.get('source')}'")
+        print(f"  All keys: {list(meta.keys())}")
+
+        # Only show first 3
+        if i >= 2:
+            break

main_chat/tests/test_retrieval.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+import sys
+
+# test_policy_retrieval.py
+
+_PROJECT_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(_PROJECT_ROOT))
+
+from main_chat.rag_pipeline.rag_retrieval import retrieve_policies
+
+# Test general policy query
+result = retrieve_policies("housing policy", k=10)
+print(f"Found {len(result['chunks'])} chunks")
+
+for i, meta in enumerate(result["metadata"][:5], 1):
+    print(f"\n{i}. {meta.get('source')}")
+    print(f"   Folder: {meta.get('folder_category')}")
+    print(f"   Preview: {result['chunks'][i-1][:150]}...")