Add llm search_vector and prospect migrations

Bump version to 2.1.9 and add full-text search support to the llm table: include search_vector in SELECTs, generate it on insert using to_tsvector('english', prompt || ' ' || completion), and add a migration + runner to create the tsvector column and GIN index. Update prospects endpoint to return the llm search_vector and adjust result field mapping. Add several SQL migration files to drop unused columns from the prospects table (cleaned, primary_email_catchall_status, primary_intent_score, primary_intent_topic, qualify_contact, secondary_intent_score, secondary_intent_topic, tertiary_email, tertiary_email_source, tertiary_email_status, tertiary_email_verification_source) and include small runner scripts for removing tertiary_email and its source.

Author: goldlabelapps

app/__init__.py

@@ -1,4 +1,4 @@
 """Python - FastAPI, Postgres, tsvector"""
 
 # Current Version
-__version__ = "2.1.8"
+__version__ = "2.1.9"

app/api/llm/llm.py

@@ -21,7 +21,7 @@ def get_llm_records(
         if prospect_id is not None:
             # No pagination for single prospect_id lookup
             select_query = """
-                SELECT id, prompt, completion, duration, time, data, model, prospect_id
+                SELECT id, prompt, completion, duration, time, data, model, prospect_id, search_vector
                 FROM llm
                 WHERE prospect_id = %s
                 ORDER BY id DESC
@@ -38,6 +38,7 @@ def get_llm_records(
                     "data": row[5],
                     "model": row[6],
                     "prospect_id": row[7],
+                    "search_vector": str(row[8]) if row[8] is not None else None,
                 }
                 for row in rows
             ]
@@ -61,7 +62,7 @@ def get_llm_records(
             count_row = cur.fetchone()
             total = count_row[0] if count_row and count_row[0] is not None else 0
             cur.execute("""
-                SELECT id, prompt, completion, duration, time, data, model, prospect_id
+                SELECT id, prompt, completion, duration, time, data, model, prospect_id, search_vector
                 FROM llm
                 ORDER BY id DESC
                 LIMIT %s OFFSET %s;
@@ -76,6 +77,7 @@ def get_llm_records(
                     "data": row[5],
                     "model": row[6],
                     "prospect_id": row[7],
+                    "search_vector": str(row[8]) if row[8] is not None else None,
                 }
                 for row in cur.fetchall()
             ]
@@ -146,13 +148,14 @@ def llm_post(payload: dict) -> dict:
             data_blob = json.dumps({"version": __version__})
             conn = get_db_connection_direct()
             cur = conn.cursor()
+            # Generate tsvector from prompt and completion
             cur.execute(
                 """
-                INSERT INTO llm (prompt, completion, duration, data, model, prospect_id)
-                VALUES (%s, %s, %s, %s, %s, %s)
+                INSERT INTO llm (prompt, completion, duration, data, model, prospect_id, search_vector)
+                VALUES (%s, %s, %s, %s, %s, %s, to_tsvector('english', %s || ' ' || %s))
                 RETURNING id;
                 """,
-                (prompt, completion, duration, data_blob, used_model, prospect_id)
+                (prompt, completion, duration, data_blob, used_model, prospect_id, prompt, completion)
             )
             record_id_row = cur.fetchone()
             record_id = record_id_row[0] if record_id_row else None

app/api/llm/sql/alter_add_search_vector.sql

@@ -0,0 +1,5 @@
+-- Migration: Add search_vector tsvector column to llm table
+ALTER TABLE llm ADD COLUMN IF NOT EXISTS search_vector tsvector;
+
+-- Optional: Create a GIN index for faster search
+CREATE INDEX IF NOT EXISTS idx_llm_search_vector ON llm USING GIN(search_vector);

app/api/llm/sql/run_alter_add_search_vector.py

@@ -0,0 +1,17 @@
+import os
+import sys
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..')))
+from app.utils.db import get_db_connection_direct
+
+if __name__ == "__main__":
+    sql = """
+    ALTER TABLE llm ADD COLUMN IF NOT EXISTS search_vector tsvector;
+    CREATE INDEX IF NOT EXISTS idx_llm_search_vector ON llm USING GIN(search_vector);
+    """
+    conn = get_db_connection_direct()
+    cur = conn.cursor()
+    cur.execute(sql)
+    conn.commit()
+    cur.close()
+    conn.close()
+    print("Migration complete: search_vector column and index added to llm table.")

app/api/prospects/prospects.py

@@ -103,16 +103,15 @@ def prospects_read_one(id: int = Path(..., description="ID of the prospect to re
                     from app.utils.db import get_db_connection_direct
                     llm_conn = get_db_connection_direct()
                     llm_cur = llm_conn.cursor()
-                    llm_cur.execute("SELECT id, prompt, completion, duration, time, data, model FROM llm WHERE prospect_id = %s ORDER BY id DESC;", (id,))
+                    llm_cur.execute("SELECT id, duration, time, data, model, search_vector FROM llm WHERE prospect_id = %s ORDER BY id DESC;", (id,))
                     llm_records = [
                         {
                             "id": r[0],
-                            "prompt": r[1],
-                            "completion": r[2],
-                            "duration": r[3],
-                            "time": r[4].isoformat() if r[4] else None,
-                            "data": r[5],
-                            "model": r[6],
+                            "duration": r[1],
+                            "time": r[2].isoformat() if r[2] else None,
+                            "data": r[3],
+                            "model": r[4],
+                            "search_vector": str(r[5]) if r[5] is not None else None,
                         }
                         for r in llm_cur.fetchall()
                     ]

app/api/prospects/sql/alter_drop_cleaned.sql

@@ -0,0 +1,2 @@
+-- Migration: Remove cleaned column from prospects table
+ALTER TABLE prospects DROP COLUMN IF EXISTS cleaned;

app/api/prospects/sql/alter_drop_do_not_call.sql

@@ -0,0 +1,2 @@
+-- Migration: Remove do_not_call column from prospects table
+ALTER TABLE prospects DROP COLUMN IF EXISTS do_not_call;

app/api/prospects/sql/alter_drop_email_confidence.sql

@@ -0,0 +1,2 @@
+-- Migration: Remove email_confidence column from prospects table
+ALTER TABLE prospects DROP COLUMN IF EXISTS email_confidence;

app/api/prospects/sql/alter_drop_home_phone.sql

@@ -0,0 +1,2 @@
+-- Migration: Remove home_phone column from prospects table
+ALTER TABLE prospects DROP COLUMN IF EXISTS home_phone;

app/api/prospects/sql/alter_drop_primary_email_catchall_status.sql

@@ -0,0 +1,2 @@
+-- Migration: Remove primary_email_catchall_status column from prospects table
+ALTER TABLE prospects DROP COLUMN IF EXISTS primary_email_catchall_status;