feat: support timescale pg_textsearch as text search extension (#359)

nicoloboschi · web-flow · commit d871c3009deb · 2026-02-12T17:38:13.000+01:00
* feat: support timescale pg_textsearch as text search extension

* refactor: deduplicate text search query in retrieve_semantic_bm25_combined

Instead of maintaining 3 complete query copies (native, vchord, pg_textsearch),
now we:
- Build backend-specific parts (score_expr, order_by, where_filter)
- Use a single query template with injected backend-specific parts

This makes maintenance easier - changes to the semantic CTE or overall structure
only need to be made once.
diff --git a/docker/docker-compose/pg_textsearch/Dockerfile b/docker/docker-compose/pg_textsearch/Dockerfile
@@ -0,0 +1,32 @@
+# PostgreSQL with pgvector and pg_textsearch extensions
+# Note: pg_textsearch requires PostgreSQL 17+
+FROM postgres:17
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    postgresql-server-dev-17 \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install pgvector
+RUN cd /tmp && \
+    git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \
+    cd pgvector && \
+    make && \
+    make install
+
+# Install pg_textsearch
+RUN cd /tmp && \
+    git clone https://github.com/timescale/pg_textsearch.git && \
+    cd pg_textsearch && \
+    make && \
+    make install
+
+# Clean up source files and build dependencies
+RUN rm -rf /tmp/pgvector /tmp/pg_textsearch && \
+    apt-get purge -y --auto-remove build-essential git postgresql-server-dev-17
+
+# Ensure extensions are preloaded
+RUN echo "shared_preload_libraries = 'pg_textsearch'" >> /usr/share/postgresql/postgresql.conf.sample
diff --git a/docker/docker-compose/pg_textsearch/docker-compose.yaml b/docker/docker-compose/pg_textsearch/docker-compose.yaml
@@ -0,0 +1,91 @@
+name: hindsight
+# Docker Compose file for Hindsight with PostgreSQL and Timescale pg_textsearch
+# docker compose -f docker/docker-compose/pg_textsearch/docker-compose.yaml down && sleep 2 && docker compose -f docker/docker-compose/pg_textsearch/docker-compose.yaml up -d
+# Make sure to set the required environment variables before running:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see below in the hindsight service)
+#
+# Usage:
+# docker compose up -d
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+
+services:
+  db:
+    # Use custom PostgreSQL image with pgvector and pg_textsearch extensions
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: hindsight-db
+    restart: always
+    # Expose PostgreSQL port
+    ports:
+      - "5437:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    networks:
+      - hindsight-net
+
+  pg-textsearch-init:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - db
+    environment:
+      - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+    command: >
+      bash -c "
+        echo 'Waiting for PostgreSQL to be ready...';
+        until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do
+          echo 'PostgreSQL is unavailable - sleeping';
+          sleep 2;
+        done;
+        echo 'PostgreSQL is ready - creating hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
+        echo 'Creating extensions in hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE;';
+        echo 'Database and extensions created successfully';
+      "
+    restart: "no"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      # LLM Configuration
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Database Configuration
+      HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+
+      # Vector and Text Search Extensions
+      HINDSIGHT_API_VECTOR_EXTENSION: pgvector
+      HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_textsearch
+
+    depends_on:
+      - db
+    networks:
+      - hindsight-net
+
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:
diff --git a/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py b/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py
@@ -51,7 +51,7 @@ def _detect_vector_extension() -> str:
 
 def _detect_text_search_extension() -> str:
     """
-    Detect or validate text search extension: 'native' or 'vchord'.
+    Detect or validate text search extension: 'native', 'vchord', or 'pg_textsearch'.
     Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
     Creates the extension if needed.
     """
@@ -69,11 +69,23 @@ def _detect_text_search_extension() -> str:
                 # Extension truly doesn't exist - re-raise the error
                 raise
         return "vchord"
+    elif text_search_extension == "pg_textsearch":
+        # Create pg_textsearch extension if not exists
+        try:
+            op.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE")
+        except Exception:
+            # Extension might already exist or user lacks permissions - verify it exists
+            conn = op.get_bind()
+            result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_textsearch'")).fetchone()
+            if not result:
+                # Extension truly doesn't exist - re-raise the error
+                raise
+        return "pg_textsearch"
     elif text_search_extension == "native":
         return "native"
     else:
         raise ValueError(
-            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native' or 'vchord'"
+            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native', 'vchord', or 'pg_textsearch'"
         )
 
 
@@ -232,6 +244,12 @@ def upgrade() -> None:
             ALTER TABLE memory_units
             ADD COLUMN search_vector bm25_catalog.bm25vector
         """)
+    elif text_search_ext == "pg_textsearch":
+        # Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
+        op.execute("""
+            ALTER TABLE memory_units
+            ADD COLUMN search_vector TEXT
+        """)
     else:  # native
         # Native PostgreSQL: tsvector with automatic generation
         op.execute("""
@@ -295,6 +313,14 @@ def upgrade() -> None:
             CREATE INDEX idx_memory_units_text_search ON memory_units
             USING bm25 (search_vector bm25_catalog.bm25_ops)
         """)
+    elif text_search_ext == "pg_textsearch":
+        # Timescale pg_textsearch BM25 index on text column
+        # Note: pg_textsearch doesn't support expressions, so we index the main text column
+        op.execute("""
+            CREATE INDEX idx_memory_units_text_search ON memory_units
+            USING bm25(text)
+            WITH (text_config='english')
+        """)
     else:  # native
         # Native PostgreSQL GIN index
         op.execute("""
diff --git a/hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py b/hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py
@@ -58,7 +58,7 @@ def _detect_vector_extension() -> str:
 
 def _detect_text_search_extension() -> str:
     """
-    Detect or validate text search extension: 'native' or 'vchord'.
+    Detect or validate text search extension: 'native', 'vchord', or 'pg_textsearch'.
     Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
     Creates the extension if needed.
     """
@@ -76,11 +76,23 @@ def _detect_text_search_extension() -> str:
                 # Extension truly doesn't exist - re-raise the error
                 raise
         return "vchord"
+    elif text_search_extension == "pg_textsearch":
+        # Create pg_textsearch extension if not exists
+        try:
+            op.execute("CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE")
+        except Exception:
+            # Extension might already exist or user lacks permissions - verify it exists
+            conn = op.get_bind()
+            result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_textsearch'")).fetchone()
+            if not result:
+                # Extension truly doesn't exist - re-raise the error
+                raise
+        return "pg_textsearch"
     elif text_search_extension == "native":
         return "native"
     else:
         raise ValueError(
-            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native' or 'vchord'"
+            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native', 'vchord', or 'pg_textsearch'"
         )
 
 
@@ -146,6 +158,15 @@ def upgrade() -> None:
             CREATE INDEX idx_learnings_text_search ON {schema}learnings
             USING bm25 (search_vector bm25_catalog.bm25_ops)
         """)
+    elif text_search_ext == "pg_textsearch":
+        # Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
+        op.execute(f"""
+            ALTER TABLE {schema}learnings ADD COLUMN search_vector TEXT
+        """)
+        op.execute(f"""
+            CREATE INDEX idx_learnings_text_search ON {schema}learnings
+            USING bm25(text) WITH (text_config='english')
+        """)
     else:  # native
         # Native PostgreSQL: tsvector with automatic generation
         op.execute(f"""
@@ -204,6 +225,16 @@ def upgrade() -> None:
             CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
             USING bm25 (search_vector bm25_catalog.bm25_ops)
         """)
+    elif text_search_ext == "pg_textsearch":
+        # Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
+        op.execute(f"""
+            ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector TEXT
+        """)
+        op.execute(f"""
+            CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
+            USING bm25(content)
+            WITH (text_config='english')
+        """)
     else:  # native
         # Native PostgreSQL: tsvector with automatic generation
         op.execute(f"""
diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py
@@ -329,8 +329,8 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 # Vector extension (pgvector vs vchord)
 DEFAULT_VECTOR_EXTENSION = "pgvector"  # Options: "pgvector", "vchord"
 
-# Text search extension (native PostgreSQL vs vchord BM25)
-DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord"
+# Text search extension (native PostgreSQL, vchord BM25, or Timescale pg_textsearch)
+DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord", "pg_textsearch"
 
 # LiteLLM defaults
 DEFAULT_LITELLM_API_BASE = "http://localhost:4000"
@@ -706,7 +706,7 @@ def validate(self) -> None:
             )
 
         # Validate text_search_extension
-        valid_text_search = ("native", "vchord")
+        valid_text_search = ("native", "vchord", "pg_textsearch")
         if self.text_search_extension not in valid_text_search:
             raise ValueError(
                 f"Invalid text_search_extension: {self.text_search_extension}. Must be one of: {', '.join(valid_text_search)}"
diff --git a/hindsight-api/hindsight_api/engine/consolidation/consolidator.py b/hindsight-api/hindsight_api/engine/consolidation/consolidator.py
@@ -1030,8 +1030,9 @@ async def _create_observation_directly(
                     tokenize($3, 'llmlingua2')::bm25_catalog.bm25vector)
             RETURNING id
         """
-    else:  # native
+    else:  # native or pg_textsearch
         # Native PostgreSQL: search_vector is GENERATED ALWAYS, don't include it
+        # pg_textsearch: indexes operate on base columns directly, don't populate search_vector
         query = f"""
             INSERT INTO {fq_table("memory_units")} (
                 id, bank_id, text, fact_type, embedding, proof_count, source_memory_ids, history,
diff --git a/hindsight-api/hindsight_api/engine/retain/fact_storage.py b/hindsight-api/hindsight_api/engine/retain/fact_storage.py
@@ -97,8 +97,9 @@ async def insert_facts_batch(
             FROM input_data
             RETURNING id
         """
-    else:  # native
+    else:  # native or pg_textsearch
         # Native PostgreSQL: search_vector is GENERATED ALWAYS, don't include it
+        # pg_textsearch: indexes operate on base columns directly, don't populate search_vector
         query = f"""
             WITH input_data AS (
                 SELECT * FROM unnest(
diff --git a/hindsight-api/hindsight_api/engine/search/retrieval.py b/hindsight-api/hindsight_api/engine/search/retrieval.py
diff --git a/hindsight-api/hindsight_api/migrations.py b/hindsight-api/hindsight_api/migrations.py
diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md
