feat: Add ilike and trigram similarity search to pagination (#431)

# Why

This PR adds search capabilities to the existing cursor-based pagination system added in #422 which only supported orderBy ordering.

This adds:
1. Case-insensitive pattern matching (ILIKE) - Useful for basic text search across entity fields
2. Trigram similarity search - Provides fuzzy matching capabilities for more advanced search use cases like handling typos or finding similar text

This enables building user-facing search features while maintaining the benefits of cursor-based pagination (stable results, efficient queries).

# How

The implementation introduces a unified PaginationSpecification that supports three strategies:

  1. Standard pagination - The existing orderBy-based pagination
  2. ILIKE search - Pattern matching with automatic wildcard escaping
  3. Trigram search - PostgreSQL trigram similarity with configurable threshold

  Key implementation details:
  - Search terms are properly parameterized to prevent SQL injection
  - ILIKE special characters (%, _, \) are escaped to prevent pattern injection
  - Results maintain stable cursor-based pagination with proper ordering:
    - ILIKE: Ordered by search fields, then ID
    - Trigram: Ordered by exact match priority, similarity score, optional extra fields, then ID

# Test Plan

  The PR includes comprehensive test coverage:

  - Unit tests for both AuthorizationResultBasedKnexEntityLoader and
  EnforcingKnexEntityLoader
  - Integration tests covering:
    - Forward/backward pagination with search
    - Multi-field search
    - Case-insensitive matching
    - Trigram similarity with various thresholds
    - Cursor stability across pages
    - Edge cases (empty results, special characters)
    - Combined with WHERE clauses
    - Security validation ensuring proper escaping and parameterization

All existing tests pass, confirming backward compatibility for standard pagination when using the new API.

Author: wschurman

packages/entity-database-adapter-knex/src/AuthorizationResultBasedKnexEntityLoader.ts

@@ -14,6 +14,7 @@ import {
   OrderByOrdering,
 } from './BasePostgresEntityDatabaseAdapter';
 import { BaseSQLQueryBuilder } from './BaseSQLQueryBuilder';
+import { PaginationStrategy } from './PaginationStrategy';
 import { SQLFragment } from './SQLOperator';
 import type { Connection, PageInfo } from './internal/EntityKnexDataManager';
 import { EntityKnexDataManager } from './internal/EntityKnexDataManager';
@@ -76,10 +77,94 @@ export interface EntityLoaderQuerySelectionModifiersWithOrderByFragment<
   orderByFragment?: SQLFragment;
 }
 
+interface SearchSpecificationBase<
+  TFields extends Record<string, any>,
+  TSelectedFields extends keyof TFields,
+> {
+  /**
+   * The search term to search for. Must be a non-empty string.
+   */
+  term: string;
+
+  /**
+   * The fields to search within. Must be a non-empty array.
+   */
+  fields: TSelectedFields[];
+}
+
+interface ILikeSearchSpecification<
+  TFields extends Record<string, any>,
+  TSelectedFields extends keyof TFields,
+> extends SearchSpecificationBase<TFields, TSelectedFields> {
+  /**
+   * Case-insensitive pattern matching search using SQL ILIKE operator.
+   * Results are ordered by the fields being searched within in the order specified, then by ID for tie-breaking and stable pagination.
+   */
+  strategy: PaginationStrategy.ILIKE_SEARCH;
+}
+
+interface TrigramSearchSpecification<
+  TFields extends Record<string, any>,
+  TSelectedFields extends keyof TFields,
+> extends SearchSpecificationBase<TFields, TSelectedFields> {
+  /**
+   * Similarity search using PostgreSQL trigram similarity. Results are ordered by exact match priority, then by similarity score, then by specified extra order by fields if provided, then by ID for tie-breaking and stable pagination.
+   * Note that trigram similarity search can be significantly slower than ILIKE search, especially on large datasets without appropriate indexes, and results may not be as relevant as more advanced full-text search solutions.
+   * It is recommended to use this strategy only when ILIKE search does not meet the application's needs and to ensure appropriate database indexing for performance.
+   */
+  strategy: PaginationStrategy.TRIGRAM_SEARCH;
+
+  /**
+   * Similarity threshold for trigram matching.
+   * Must be between 0 and 1, where:
+   * - 0 matches everything
+   * - 1 requires exact match
+   *
+   * Recommended threshold values:
+   * - 0.3: Loose matching, allows more variation (default PostgreSQL similarity threshold)
+   * - 0.4-0.5: Moderate matching, good balance for most use cases
+   * - 0.6+: Strict matching, requires high similarity
+   */
+  threshold: number;
+
+  /**
+   * Optional additional fields to order by after similarity score and before ID for tie-breaking.
+   * These fields are independent of search fields and can be used to provide meaningful
+   * ordering when multiple results have the same similarity score.
+   */
+  extraOrderByFields?: TSelectedFields[];
+}
+
+interface StandardPaginationSpecification<
+  TFields extends Record<string, any>,
+  TSelectedFields extends keyof TFields,
+> {
+  /**
+   * Standard pagination without search. Results are ordered by the specified orderBy fields.
+   */
+  strategy: PaginationStrategy.STANDARD;
+
+  /**
+   * Order the entities by specified columns and orders. If the ID field is not included, it will be automatically added for stable pagination.
+   */
+  orderBy: EntityLoaderOrderByClause<TFields, TSelectedFields>[];
+}
+
 /**
- * Base pagination arguments
+ * Pagination specification for SQL-based pagination (with or without search).
  */
-interface EntityLoaderBasePaginationArgs<
+export type PaginationSpecification<
+  TFields extends Record<string, any>,
+  TSelectedFields extends keyof TFields,
+> =
+  | StandardPaginationSpecification<TFields, TSelectedFields>
+  | ILikeSearchSpecification<TFields, TSelectedFields>
+  | TrigramSearchSpecification<TFields, TSelectedFields>;
+
+/**
+ * Base unified pagination arguments
+ */
+interface EntityLoaderBaseUnifiedPaginationArgs<
   TFields extends Record<string, any>,
   TSelectedFields extends keyof TFields,
 > {
@@ -89,56 +174,56 @@ interface EntityLoaderBasePaginationArgs<
   where?: SQLFragment;
 
   /**
-   * Order the entities by specified columns and orders. If the ID field is not included in the orderBy, it will be automatically included as the last orderBy field to ensure stable pagination.
+   * Pagination specification determining how to order and paginate results.
    */
-  orderBy?: EntityLoaderOrderByClause<TFields, TSelectedFields>[];
+  pagination: PaginationSpecification<TFields, TSelectedFields>;
 }
 
 /**
- * Forward pagination arguments
+ * Forward unified pagination arguments
  */
-export interface EntityLoaderForwardPaginationArgs<
+export interface EntityLoaderForwardUnifiedPaginationArgs<
   TFields extends Record<string, any>,
   TSelectedFields extends keyof TFields,
-> extends EntityLoaderBasePaginationArgs<TFields, TSelectedFields> {
+> extends EntityLoaderBaseUnifiedPaginationArgs<TFields, TSelectedFields> {
   /**
-   * The number of entities to return starting from the entity after the cursor (for forward pagination). Must be a positive integer.
+   * The number of entities to return starting from the entity after the cursor. Must be a positive integer.
    */
   first: number;
 
   /**
-   * The cursor to paginate after for forward pagination, typically an opaque string encoding of the values of the cursor fields of the last entity in the previous page. If not provided, pagination starts from the beginning of the result set.
+   * The cursor to paginate after for forward pagination.
    */
   after?: string;
 }
 
 /**
- * Backward pagination arguments
+ * Backward unified pagination arguments
  */
-export interface EntityLoaderBackwardPaginationArgs<
+export interface EntityLoaderBackwardUnifiedPaginationArgs<
   TFields extends Record<string, any>,
   TSelectedFields extends keyof TFields,
-> extends EntityLoaderBasePaginationArgs<TFields, TSelectedFields> {
+> extends EntityLoaderBaseUnifiedPaginationArgs<TFields, TSelectedFields> {
   /**
-   * The number of entities to return starting from the entity before the cursor (for backward pagination). Must be a positive integer.
+   * The number of entities to return starting from the entity before the cursor. Must be a positive integer.
    */
   last: number;
 
   /**
-   * The cursor to paginate before for backward pagination, typically an opaque string encoding of the values of the cursor fields of the first entity in the previous page. If not provided, pagination starts from the end of the result set.
+   * The cursor to paginate before for backward pagination.
    */
   before?: string;
 }
 
 /**
- * Load page pagination arguments, which can be either forward or backward pagination arguments.
+ * Load page pagination arguments, which can be either forward or backward unified pagination arguments.
  */
 export type EntityLoaderLoadPageArgs<
   TFields extends Record<string, any>,
   TSelectedFields extends keyof TFields,
 > =
-  | EntityLoaderForwardPaginationArgs<TFields, TSelectedFields>
-  | EntityLoaderBackwardPaginationArgs<TFields, TSelectedFields>;
+  | EntityLoaderForwardUnifiedPaginationArgs<TFields, TSelectedFields>
+  | EntityLoaderBackwardUnifiedPaginationArgs<TFields, TSelectedFields>;
 
 /**
  * Authorization-result-based knex entity loader for non-data-loader-based load methods.
@@ -267,18 +352,15 @@ export class AuthorizationResultBasedKnexEntityLoader<
   }
 
   /**
-   * Load a page of entities with Relay-style cursor pagination.
+   * Load a page of entities with Relay-style cursor pagination using a unified pagination specification.
    * Only returns successfully authorized entities for cursor stability; failed authorization results are filtered out.
    *
    * @returns Connection with only successfully authorized entities
    */
-  async loadPageBySQLAsync(
+  async loadPageAsync(
     args: EntityLoaderLoadPageArgs<TFields, TSelectedFields>,
   ): Promise<Connection<TEntity>> {
-    const pageResult = await this.knexDataManager.loadPageBySQLFragmentAsync(
-      this.queryContext,
-      args,
-    );
+    const pageResult = await this.knexDataManager.loadPageAsync(this.queryContext, args);
 
     const edgeResults = await Promise.all(
       pageResult.edges.map(async (edge) => {

packages/entity-database-adapter-knex/src/EnforcingKnexEntityLoader.ts

@@ -191,37 +191,14 @@ export class EnforcingKnexEntityLoader<
   /**
    * Load a page of entities with Relay-style cursor pagination.
    *
-   * @param args - Pagination arguments with either first/after or last/before
-   *
-   * @example
-   * ```typescript
-   * // Forward pagination - get first 10 items
-   * const users = await TestEntity.knexLoader(vc)
-   *   .loadPageBySQLAsync({
-   *     first: 10,
-   *     where: sql`age > ${18}`,
-   *     orderBy: 'created_at'
-   *   });
-   *
-   * // Backward pagination with cursor - get last 10 items before the cursor
-   * const lastResults = await TestEntity.knexLoader(vc)
-   *   .loadPageBySQLAsync({
-   *     last: 10,
-   *     where: sql`status = ${'active'}`,
-   *     before: cursor,
-   *   });
-   * ```
-   *
+   * @param args - Pagination arguments with pagination and either first/after or last/before
+   * @returns a page of entities matching the pagination arguments
    * @throws EntityNotAuthorizedError if viewer is not authorized to view any returned entity
    */
-  async loadPageBySQLAsync(
+  async loadPageAsync(
     args: EntityLoaderLoadPageArgs<TFields, TSelectedFields>,
   ): Promise<Connection<TEntity>> {
-    const pageResult = await this.knexDataManager.loadPageBySQLFragmentAsync(
-      this.queryContext,
-      args,
-    );
-
+    const pageResult = await this.knexDataManager.loadPageAsync(this.queryContext, args);
     const edges = await Promise.all(
       pageResult.edges.map(async (edge) => {
         const entityResult = await this.constructionUtils.constructAndAuthorizeEntityAsync(

packages/entity-database-adapter-knex/src/PaginationStrategy.ts

@@ -0,0 +1,32 @@
+/**
+ * Search strategy for SQL-based pagination.
+ */
+export enum PaginationStrategy {
+  /**
+   * Standard pagination with ORDER BY. Results are ordered by the specified orderBy fields, with ID field automatically included for stable pagination if not already present.
+   */
+  STANDARD = 'standard',
+
+  /**
+   * Case-insensitive pattern matching search using SQL ILIKE operator.
+   * Results are ordered by the fields being searched within in the order specified, then by ID for tie-breaking and stable pagination.
+   */
+  ILIKE_SEARCH = 'ilike-search',
+
+  /**
+   * Similarity search using PostgreSQL trigram similarity. Results are ordered by exact match priority, then by similarity score, then by specified extra order by fields if provided, then by ID for tie-breaking and stable pagination.
+   *
+   * Performance considerations:
+   * - Trigram search can be significantly slower than ILIKE search, especially on large datasets without appropriate indexes.
+   * - Consider using ILIKE search for smaller datasets or when exact substring matching is sufficient
+   * - For larger datasets, ensure proper indexing or consider dedicated full-text search solutions.
+   * - For optimal performance, create GIN or GIST indexes on searchable columns:
+   * ```sql
+   *   CREATE EXTENSION IF NOT EXISTS pg_trgm;
+   *   CREATE INDEX idx_table_field_trigram ON table_name USING gin(field_name gin_trgm_ops);
+   *   -- Or for multiple columns:
+   *   CREATE INDEX idx_table_search ON table_name USING gin((field1 || ' ' || field2) gin_trgm_ops);
+   * ```
+   */
+  TRIGRAM_SEARCH = 'trigram',
+}

packages/entity-database-adapter-knex/src/PostgresEntityDatabaseAdapter.ts

@@ -110,14 +110,19 @@ export class PostgresEntityDatabaseAdapter<
     query: Knex.QueryBuilder,
     querySelectionModifiers: TableQuerySelectionModifiersWithOrderByRaw,
   ): Knex.QueryBuilder {
-    let ret = this.applyQueryModifiersToQuery(query, querySelectionModifiers);
-
     const { orderByRaw } = querySelectionModifiers;
+
+    // orderByRaw takes precedence over orderBy - they are mutually exclusive
     if (orderByRaw !== undefined) {
-      ret = ret.orderByRaw(orderByRaw);
+      // Apply only orderByRaw (offset/limit still applied, but not orderBy)
+      return this.applyQueryModifiersToQuery(query, {
+        ...querySelectionModifiers,
+        orderBy: undefined, // Explicitly exclude orderBy when orderByRaw is present
+      }).orderByRaw(orderByRaw);
+    } else {
+      // Apply regular orderBy (and offset/limit)
+      return this.applyQueryModifiersToQuery(query, querySelectionModifiers);
     }
-
-    return ret;
   }
 
   private applyQueryModifiersToQuery(
@@ -216,10 +221,19 @@ export class PostgresEntityDatabaseAdapter<
       .select()
       .from(tableName)
       .whereRaw(sqlFragment.sql, sqlFragment.getKnexBindings());
-    query = this.applyQueryModifiersToQuery(query, querySelectionModifiers);
+
+    // Apply order by modifiers
+    // orderByFragment takes precedence over orderBy - they are mutually exclusive
     const { orderByFragment } = querySelectionModifiers;
     if (orderByFragment !== undefined) {
-      query = query.orderByRaw(orderByFragment.sql, orderByFragment.getKnexBindings());
+      // Apply only orderByFragment (offset/limit still applied, but not orderBy)
+      query = this.applyQueryModifiersToQuery(query, {
+        ...querySelectionModifiers,
+        orderBy: undefined, // Explicitly exclude orderBy when orderByFragment is present
+      }).orderByRaw(orderByFragment.sql, orderByFragment.getKnexBindings());
+    } else {
+      // Apply regular orderBy (and offset/limit)
+      query = this.applyQueryModifiersToQuery(query, querySelectionModifiers);
     }
     return await wrapNativePostgresCallAsync(() => query);
   }