feat: Add ilike and trigram similarity search to pagination

wschurman · wschurman · commit ec9eae375dad · 2026-02-10T15:58:42.000-07:00
diff --git a/packages/entity-database-adapter-knex/src/AuthorizationResultBasedKnexEntityLoader.ts b/packages/entity-database-adapter-knex/src/AuthorizationResultBasedKnexEntityLoader.ts
@@ -76,6 +76,39 @@ export interface EntityLoaderQuerySelectionModifiersWithOrderByFragment<
   orderByFragment?: SQLFragment;
 }
 
+/**
+ * Search strategy for SQL-based pagination.
+ */
+export enum SearchStrategy {
+  ILIKE = 'ilike',
+  TRIGRAM = 'trigram',
+}
+
+interface SearchSpecificationBase<TFields extends Record<string, any>> {
+  term: string;
+  fields: (keyof TFields)[];
+}
+
+interface ILikeSearchSpecification<
+  TFields extends Record<string, any>,
+> extends SearchSpecificationBase<TFields> {
+  strategy: SearchStrategy.ILIKE;
+}
+
+interface TrigramSearchSpecification<
+  TFields extends Record<string, any>,
+> extends SearchSpecificationBase<TFields> {
+  strategy: SearchStrategy.TRIGRAM;
+  threshold: number;
+}
+
+/**
+ * Search specification for SQL-based pagination.
+ */
+export type SearchSpecification<TFields extends Record<string, any>> =
+  | ILikeSearchSpecification<TFields>
+  | TrigramSearchSpecification<TFields>;
+
 /**
  * Base pagination arguments
  */
@@ -104,6 +137,12 @@ interface EntityLoaderBasePaginationArgs<
    * The total count is available in the connection's totalCount field.
    */
   includeTotal?: boolean;
+
+  /**
+   * Search specification for filtering entities by text search.
+   * Supports ILIKE pattern matching or PostgreSQL trigram similarity search.
+   */
+  search?: SearchSpecification<TFields>;
 }
 
 /**
diff --git a/packages/entity-database-adapter-knex/src/__integration-tests__/PostgresEntityIntegration-test.ts b/packages/entity-database-adapter-knex/src/__integration-tests__/PostgresEntityIntegration-test.ts
@@ -10,6 +10,7 @@ import { knex, Knex } from 'knex';
 import nullthrows from 'nullthrows';
 import { setTimeout } from 'timers/promises';
 
+import { SearchStrategy } from '../AuthorizationResultBasedKnexEntityLoader';
 import { OrderByOrdering } from '../BasePostgresEntityDatabaseAdapter';
 import { raw, sql, SQLFragment, SQLFragmentHelpers } from '../SQLOperator';
 import { PostgresTestEntity } from '../__testfixtures__/PostgresTestEntity';
@@ -1913,5 +1914,182 @@ describe('postgres entity integration', () => {
       expect(emptyBackwardPageWithTotal.pageInfo.startCursor).toBeNull();
       expect(emptyBackwardPageWithTotal.pageInfo.endCursor).toBeNull();
     });
+
+    it('supports search with ILIKE strategy', async () => {
+      const vc = new ViewerContext(createKnexIntegrationTestEntityCompanionProvider(knexInstance));
+
+      await PostgresTestEntity.dropPostgresTableAsync(knexInstance);
+      await PostgresTestEntity.createOrTruncatePostgresTableAsync(knexInstance);
+
+      // Create test data with searchable names
+      const names = [
+        'Alice Johnson',
+        'Bob Smith',
+        'Charlie Brown',
+        'David Smith',
+        'Eve Johnson',
+        'Frank Miller',
+      ];
+      for (let i = 0; i < names.length; i++) {
+        await PostgresTestEntity.creator(vc)
+          .setField('name', names[i]!)
+          .setField('hasACat', i % 2 === 0)
+          .createAsync();
+      }
+
+      // Search for names containing "Johnson"
+      const searchResults = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 10,
+        search: {
+          strategy: SearchStrategy.ILIKE,
+          term: 'Johnson',
+          fields: ['name'],
+        },
+        orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
+      });
+
+      expect(searchResults.edges).toHaveLength(2);
+      expect(searchResults.edges[0]?.node.getField('name')).toBe('Alice Johnson');
+      expect(searchResults.edges[1]?.node.getField('name')).toBe('Eve Johnson');
+
+      // Search for names containing "Smith" with pagination
+      const smithPage1 = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 1,
+        search: {
+          strategy: SearchStrategy.ILIKE,
+          term: 'Smith',
+          fields: ['name'],
+        },
+        orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
+      });
+
+      expect(smithPage1.edges).toHaveLength(1);
+      expect(smithPage1.edges[0]?.node.getField('name')).toBe('Bob Smith');
+      expect(smithPage1.pageInfo.hasNextPage).toBe(true);
+
+      // Get next page
+      const smithPage2 = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 1,
+        after: smithPage1.pageInfo.endCursor!,
+        search: {
+          strategy: SearchStrategy.ILIKE,
+          term: 'Smith',
+          fields: ['name'],
+        },
+        orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
+      });
+
+      expect(smithPage2.edges).toHaveLength(1);
+      expect(smithPage2.edges[0]?.node.getField('name')).toBe('David Smith');
+      expect(smithPage2.pageInfo.hasNextPage).toBe(false);
+
+      // Test partial match (case insensitive)
+      const partialMatch = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 10,
+        search: {
+          strategy: SearchStrategy.ILIKE,
+          term: 'john',
+          fields: ['name'],
+        },
+        orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
+      });
+
+      expect(partialMatch.edges).toHaveLength(2);
+      expect(partialMatch.edges[0]?.node.getField('name')).toBe('Alice Johnson');
+      expect(partialMatch.edges[1]?.node.getField('name')).toBe('Eve Johnson');
+
+      // Test search with WHERE clause
+      const combinedFilter = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 10,
+        where: sql`has_a_cat = ${true}`,
+        search: {
+          strategy: SearchStrategy.ILIKE,
+          term: 'Johnson',
+          fields: ['name'],
+        },
+        orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
+      });
+
+      // Both Alice Johnson (index 0) and Eve Johnson (index 4) have cats
+      expect(combinedFilter.edges).toHaveLength(2);
+      expect(combinedFilter.edges[0]?.node.getField('name')).toBe('Alice Johnson');
+      expect(combinedFilter.edges[0]?.node.getField('hasACat')).toBe(true);
+      expect(combinedFilter.edges[1]?.node.getField('name')).toBe('Eve Johnson');
+      expect(combinedFilter.edges[1]?.node.getField('hasACat')).toBe(true);
+
+      // Test search with includeTotal
+      const withTotal = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 1,
+        search: {
+          strategy: SearchStrategy.ILIKE,
+          term: 'Smith',
+          fields: ['name'],
+        },
+        orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
+        includeTotal: true,
+      });
+
+      expect(withTotal.edges).toHaveLength(1);
+      expect(withTotal.totalCount).toBe(2); // Bob Smith and David Smith
+    });
+
+    it('supports trigram similarity search', async () => {
+      const vc = new ViewerContext(createKnexIntegrationTestEntityCompanionProvider(knexInstance));
+
+      await PostgresTestEntity.dropPostgresTableAsync(knexInstance);
+      await PostgresTestEntity.createOrTruncatePostgresTableAsync(knexInstance);
+
+      // Enable pg_trgm extension for trigram similarity
+      await knexInstance.raw('CREATE EXTENSION IF NOT EXISTS pg_trgm');
+
+      // Create test data with similar names
+      const names = ['Johnson', 'Jonson', 'Johnsen', 'Smith', 'Smyth', 'Schmidt'];
+      for (let i = 0; i < names.length; i++) {
+        await PostgresTestEntity.creator(vc)
+          .setField('name', names[i]!)
+          .setField('hasACat', i < 3) // First 3 have cats
+          .createAsync();
+      }
+
+      // Search for similar names to "Johnson" using trigram
+      const trigramSearch = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 10,
+        search: {
+          strategy: SearchStrategy.TRIGRAM,
+          term: 'Johnson',
+          fields: ['name'],
+          threshold: 0.3, // Similarity threshold
+        },
+      });
+
+      // Should find exact match and similar names, ordered by relevance
+      expect(trigramSearch.edges.length).toBeGreaterThan(0);
+      // Exact match should come first due to ILIKE matching
+      expect(trigramSearch.edges[0]?.node.getField('name')).toBe('Johnson');
+
+      // The similar names (Jonson, Johnsen) should also be included
+      const foundNames = trigramSearch.edges.map((e) => e.node.getField('name'));
+      expect(foundNames).toContain('Jonson');
+      expect(foundNames).toContain('Johnsen');
+
+      // Test combining with WHERE clause
+      const filteredTrigram = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
+        first: 10,
+        where: sql`has_a_cat = ${true}`,
+        search: {
+          strategy: SearchStrategy.TRIGRAM,
+          term: 'Johnson',
+          fields: ['name'],
+          threshold: 0.3,
+        },
+      });
+
+      // Only the Johnson-like names with cats
+      expect(filteredTrigram.edges.length).toBeGreaterThan(0);
+      expect(filteredTrigram.edges.length).toBeLessThanOrEqual(3);
+      filteredTrigram.edges.forEach((edge) => {
+        expect(edge.node.getField('hasACat')).toBe(true);
+      });
+    });
   });
 });
diff --git a/packages/entity-database-adapter-knex/src/internal/EntityKnexDataManager.ts b/packages/entity-database-adapter-knex/src/internal/EntityKnexDataManager.ts
@@ -10,6 +10,10 @@ import {
 } from '@expo/entity';
 import assert from 'assert';
 
+import {
+  SearchStrategy,
+  type SearchSpecification,
+} from '../AuthorizationResultBasedKnexEntityLoader';
 import {
   BasePostgresEntityDatabaseAdapter,
   FieldEqualityCondition,
@@ -29,6 +33,7 @@ interface BasePaginationArgs<TFields extends Record<string, any>> {
   orderBy?: PostgresOrderByClause<TFields>[];
   cursorFields?: readonly (keyof TFields)[];
   includeTotal?: boolean;
+  search?: SearchSpecification<TFields>;
 }
 
 /**
@@ -211,7 +216,7 @@ export class EntityKnexDataManager<
     }
 
     const isForward = 'first' in args;
-    const { where, orderBy, cursorFields, includeTotal } = args;
+    const { where, orderBy, cursorFields, includeTotal, search } = args;
 
     let limit: number;
     let cursor: string | undefined;
@@ -235,9 +240,26 @@ export class EntityKnexDataManager<
     // Decode cursor
     const decodedCursor = cursor ? this.decodeCursor(cursor, finalCursorFields) : null;
 
+    // Build search conditions if search is provided
+    let searchWhere: SQLFragment | undefined;
+    let searchOrderByFragment: SQLFragment | undefined;
+    if (search) {
+      const searchConditions = this.buildSearchConditionAndOrderBy(search);
+      searchWhere = searchConditions.searchWhere;
+      searchOrderByFragment = searchConditions.searchOrderByFragment;
+    }
+
+    // Combine WHERE conditions: base where + search where + cursor condition
+    const combinedWhere = [where, searchWhere]
+      .filter(Boolean)
+      .reduce<SQLFragment | undefined>((acc, condition) => {
+        if (!acc) return condition;
+        return sql`(${acc}) AND (${condition})`;
+      }, undefined);
+
     // Build WHERE clause with cursor condition for keyset pagination
     const whereClause = this.buildWhereClause({
-      ...(where && { where }),
+      ...(combinedWhere && { where: combinedWhere }),
       cursorFields: finalCursorFields,
       decodedCursor,
       direction: isForward ? 'forward' : 'backward',
@@ -288,7 +310,10 @@ export class EntityKnexDataManager<
           queryContext,
         )(
           this.databaseAdapter.fetchManyBySQLFragmentAsync(queryContext, whereClause, {
-            orderBy: finalOrderByClauses,
+            // Use orderByFragment if search provides one, otherwise use regular orderBy
+            ...(searchOrderByFragment
+              ? { orderByFragment: searchOrderByFragment }
+              : { orderBy: finalOrderByClauses }),
             limit: limit + 1,
           }),
         ),
@@ -298,7 +323,12 @@ export class EntityKnexDataManager<
           this.entityClassName,
           queryContext,
           () => 1,
-        )(this.databaseAdapter.fetchCountBySQLFragmentAsync(queryContext, where ?? sql`1 = 1`)),
+        )(
+          this.databaseAdapter.fetchCountBySQLFragmentAsync(
+            queryContext,
+            combinedWhere ?? sql`1 = 1`,
+          ),
+        ),
       ]);
       fieldObjects = fieldObjectsResult;
       totalCount = totalCountResult;
@@ -443,4 +473,61 @@ export class EntityKnexDataManager<
     )})
   `;
   }
+
+  private buildSearchConditionAndOrderBy(search: SearchSpecification<TFields>): {
+    searchWhere: SQLFragment;
+    searchOrderByFragment: SQLFragment | undefined;
+  } {
+    switch (search.strategy) {
+      case SearchStrategy.ILIKE: {
+        // Simple ILIKE search
+        const conditions = search.fields.map((field) => {
+          const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
+          return sql`${identifier(dbField)} ILIKE ${'%' + search.term + '%'}`;
+        });
+        return {
+          searchWhere: conditions.length > 0 ? SQLFragment.join(conditions, ' OR ') : sql`1 = 0`,
+          searchOrderByFragment: undefined,
+        };
+      }
+
+      case SearchStrategy.TRIGRAM: {
+        // PostgreSQL trigram similarity
+        const conditions = search.fields.map((field) => {
+          const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
+          return sql`similarity(${identifier(dbField)}, ${search.term}) > ${search.threshold}`;
+        });
+        const ilikeConditions = search.fields.map((field) => {
+          const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
+          return sql`${identifier(dbField)} ILIKE ${'%' + search.term + '%'}`;
+        });
+        // Combine exact matches (ILIKE) with similarity
+        const allConditions = [...ilikeConditions, ...conditions];
+        const idField = getDatabaseFieldForEntityField(
+          this.entityConfiguration,
+          this.entityConfiguration.idField,
+        );
+        return {
+          searchWhere: SQLFragment.join(allConditions, ' OR '),
+          // For trigram search, order by relevance
+          // 1. Exact matches first (ILIKE)
+          // 2. Then by similarity score
+          // 3. Then by ID field
+          searchOrderByFragment: sql`CASE WHEN ${SQLFragment.join(
+            search.fields.map((field) => {
+              const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
+              return sql`${identifier(dbField)} ILIKE ${'%' + search.term + '%'}`;
+            }),
+            ' OR ',
+          )} THEN 0 ELSE 1 END, GREATEST(${SQLFragment.join(
+            search.fields.map((field) => {
+              const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
+              return sql`similarity(${identifier(dbField)}, ${search.term})`;
+            }),
+            ', ',
+          )}) DESC, ${identifier(idField)} DESC`,
+        };
+      }
+    }
+  }
 }
