Skip to content

Commit 6a2d709

Browse files
wschurmanwschurman
committed
feat: Add ilike and trigram similarity search to pagination
1 parent 58deffd commit 6a2d709

10 files changed

Lines changed: 2475 additions & 508 deletions

‎packages/entity-database-adapter-knex/src/AuthorizationResultBasedKnexEntityLoader.ts‎

Lines changed: 105 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
OrderByOrdering,
1515
} from './BasePostgresEntityDatabaseAdapter';
1616
import { BaseSQLQueryBuilder } from './BaseSQLQueryBuilder';
17+
import { PaginationStrategy } from './PaginationStrategy';
1718
import { SQLFragment } from './SQLOperator';
1819
import type { Connection, PageInfo } from './internal/EntityKnexDataManager';
1920
import { EntityKnexDataManager } from './internal/EntityKnexDataManager';
@@ -76,10 +77,94 @@ export interface EntityLoaderQuerySelectionModifiersWithOrderByFragment<
7677
orderByFragment?: SQLFragment;
7778
}
7879

80+
interface SearchSpecificationBase<
81+
TFields extends Record<string, any>,
82+
TSelectedFields extends keyof TFields,
83+
> {
84+
/**
85+
* The search term to search for. Must be a non-empty string.
86+
*/
87+
term: string;
88+
89+
/**
90+
* The fields to search within. Must be a non-empty array.
91+
*/
92+
fields: TSelectedFields[];
93+
}
94+
95+
interface ILikeSearchSpecification<
96+
TFields extends Record<string, any>,
97+
TSelectedFields extends keyof TFields,
98+
> extends SearchSpecificationBase<TFields, TSelectedFields> {
99+
/**
100+
* Case-insensitive pattern matching search using SQL ILIKE operator.
101+
* Results are ordered by the fields being searched within in the order specified, then by ID for tie-breaking and stable pagination.
102+
*/
103+
strategy: PaginationStrategy.ILIKE_SEARCH;
104+
}
105+
106+
interface TrigramSearchSpecification<
107+
TFields extends Record<string, any>,
108+
TSelectedFields extends keyof TFields,
109+
> extends SearchSpecificationBase<TFields, TSelectedFields> {
110+
/**
111+
* Similarity search using PostgreSQL trigram similarity. Results are ordered by exact match priority, then by similarity score, then by specified extra order by fields if provided, then by ID for tie-breaking and stable pagination.
112+
* Note that trigram similarity search can be significantly slower than ILIKE search, especially on large datasets without appropriate indexes, and results may not be as relevant as more advanced full-text search solutions.
113+
* It is recommended to use this strategy only when ILIKE search does not meet the application's needs and to ensure appropriate database indexing for performance.
114+
*/
115+
strategy: PaginationStrategy.TRIGRAM_SEARCH;
116+
117+
/**
118+
* Similarity threshold for trigram matching.
119+
* Must be between 0 and 1, where:
120+
* - 0 matches everything
121+
* - 1 requires exact match
122+
*
123+
* Recommended threshold values:
124+
* - 0.3: Loose matching, allows more variation (default PostgreSQL similarity threshold)
125+
* - 0.4-0.5: Moderate matching, good balance for most use cases
126+
* - 0.6+: Strict matching, requires high similarity
127+
*/
128+
threshold: number;
129+
130+
/**
131+
* Optional additional fields to order by after similarity score and before ID for tie-breaking.
132+
* These fields are independent of search fields and can be used to provide meaningful
133+
* ordering when multiple results have the same similarity score.
134+
*/
135+
extraOrderByFields?: TSelectedFields[];
136+
}
137+
138+
interface StandardPaginationSpecification<
139+
TFields extends Record<string, any>,
140+
TSelectedFields extends keyof TFields,
141+
> {
142+
/**
143+
* Standard pagination without search. Results are ordered by the specified orderBy fields.
144+
*/
145+
strategy: PaginationStrategy.STANDARD;
146+
147+
/**
148+
* Order the entities by specified columns and orders. If the ID field is not included, it will be automatically added for stable pagination.
149+
*/
150+
orderBy: EntityLoaderOrderByClause<TFields, TSelectedFields>[];
151+
}
152+
79153
/**
80-
* Base pagination arguments
154+
* Pagination specification for SQL-based pagination (with or without search).
81155
*/
82-
interface EntityLoaderBasePaginationArgs<
156+
export type PaginationSpecification<
157+
TFields extends Record<string, any>,
158+
TSelectedFields extends keyof TFields,
159+
> =
160+
| StandardPaginationSpecification<TFields, TSelectedFields>
161+
| ILikeSearchSpecification<TFields, TSelectedFields>
162+
| TrigramSearchSpecification<TFields, TSelectedFields>;
163+
164+
/**
165+
* Base unified pagination arguments
166+
*/
167+
interface EntityLoaderBaseUnifiedPaginationArgs<
83168
TFields extends Record<string, any>,
84169
TSelectedFields extends keyof TFields,
85170
> {
@@ -89,56 +174,56 @@ interface EntityLoaderBasePaginationArgs<
89174
where?: SQLFragment;
90175

91176
/**
92-
* Order the entities by specified columns and orders. If the ID field is not included in the orderBy, it will be automatically included as the last orderBy field to ensure stable pagination.
177+
* Pagination specification determining how to order and paginate results.
93178
*/
94-
orderBy?: EntityLoaderOrderByClause<TFields, TSelectedFields>[];
179+
pagination: PaginationSpecification<TFields, TSelectedFields>;
95180
}
96181

97182
/**
98-
* Forward pagination arguments
183+
* Forward unified pagination arguments
99184
*/
100-
export interface EntityLoaderForwardPaginationArgs<
185+
export interface EntityLoaderForwardUnifiedPaginationArgs<
101186
TFields extends Record<string, any>,
102187
TSelectedFields extends keyof TFields,
103-
> extends EntityLoaderBasePaginationArgs<TFields, TSelectedFields> {
188+
> extends EntityLoaderBaseUnifiedPaginationArgs<TFields, TSelectedFields> {
104189
/**
105-
* The number of entities to return starting from the entity after the cursor (for forward pagination). Must be a positive integer.
190+
* The number of entities to return starting from the entity after the cursor. Must be a positive integer.
106191
*/
107192
first: number;
108193

109194
/**
110-
* The cursor to paginate after for forward pagination, typically an opaque string encoding of the values of the cursor fields of the last entity in the previous page. If not provided, pagination starts from the beginning of the result set.
195+
* The cursor to paginate after for forward pagination.
111196
*/
112197
after?: string;
113198
}
114199

115200
/**
116-
* Backward pagination arguments
201+
* Backward unified pagination arguments
117202
*/
118-
export interface EntityLoaderBackwardPaginationArgs<
203+
export interface EntityLoaderBackwardUnifiedPaginationArgs<
119204
TFields extends Record<string, any>,
120205
TSelectedFields extends keyof TFields,
121-
> extends EntityLoaderBasePaginationArgs<TFields, TSelectedFields> {
206+
> extends EntityLoaderBaseUnifiedPaginationArgs<TFields, TSelectedFields> {
122207
/**
123-
* The number of entities to return starting from the entity before the cursor (for backward pagination). Must be a positive integer.
208+
* The number of entities to return starting from the entity before the cursor. Must be a positive integer.
124209
*/
125210
last: number;
126211

127212
/**
128-
* The cursor to paginate before for backward pagination, typically an opaque string encoding of the values of the cursor fields of the first entity in the previous page. If not provided, pagination starts from the end of the result set.
213+
* The cursor to paginate before for backward pagination.
129214
*/
130215
before?: string;
131216
}
132217

133218
/**
134-
* Load page pagination arguments, which can be either forward or backward pagination arguments.
219+
* Load page pagination arguments, which can be either forward or backward unified pagination arguments.
135220
*/
136221
export type EntityLoaderLoadPageArgs<
137222
TFields extends Record<string, any>,
138223
TSelectedFields extends keyof TFields,
139224
> =
140-
| EntityLoaderForwardPaginationArgs<TFields, TSelectedFields>
141-
| EntityLoaderBackwardPaginationArgs<TFields, TSelectedFields>;
225+
| EntityLoaderForwardUnifiedPaginationArgs<TFields, TSelectedFields>
226+
| EntityLoaderBackwardUnifiedPaginationArgs<TFields, TSelectedFields>;
142227

143228
/**
144229
* Authorization-result-based knex entity loader for non-data-loader-based load methods.
@@ -267,18 +352,15 @@ export class AuthorizationResultBasedKnexEntityLoader<
267352
}
268353

269354
/**
270-
* Load a page of entities with Relay-style cursor pagination.
355+
* Load a page of entities with Relay-style cursor pagination using a unified pagination specification.
271356
* Only returns successfully authorized entities for cursor stability; failed authorization results are filtered out.
272357
*
273358
* @returns Connection with only successfully authorized entities
274359
*/
275-
async loadPageBySQLAsync(
360+
async loadPageAsync(
276361
args: EntityLoaderLoadPageArgs<TFields, TSelectedFields>,
277362
): Promise<Connection<TEntity>> {
278-
const pageResult = await this.knexDataManager.loadPageBySQLFragmentAsync(
279-
this.queryContext,
280-
args,
281-
);
363+
const pageResult = await this.knexDataManager.loadPageAsync(this.queryContext, args);
282364

283365
const edgeResults = await Promise.all(
284366
pageResult.edges.map(async (edge) => {

‎packages/entity-database-adapter-knex/src/EnforcingKnexEntityLoader.ts‎

Lines changed: 4 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -191,37 +191,14 @@ export class EnforcingKnexEntityLoader<
191191
/**
192192
* Load a page of entities with Relay-style cursor pagination.
193193
*
194-
* @param args - Pagination arguments with either first/after or last/before
195-
*
196-
* @example
197-
* ```typescript
198-
* // Forward pagination - get first 10 items
199-
* const users = await TestEntity.knexLoader(vc)
200-
* .loadPageBySQLAsync({
201-
* first: 10,
202-
* where: sql`age > ${18}`,
203-
* orderBy: 'created_at'
204-
* });
205-
*
206-
* // Backward pagination with cursor - get last 10 items before the cursor
207-
* const lastResults = await TestEntity.knexLoader(vc)
208-
* .loadPageBySQLAsync({
209-
* last: 10,
210-
* where: sql`status = ${'active'}`,
211-
* before: cursor,
212-
* });
213-
* ```
214-
*
194+
* @param args - Pagination arguments with pagination and either first/after or last/before
195+
* @returns a page of entities matching the pagination arguments
215196
* @throws EntityNotAuthorizedError if viewer is not authorized to view any returned entity
216197
*/
217-
async loadPageBySQLAsync(
198+
async loadPageAsync(
218199
args: EntityLoaderLoadPageArgs<TFields, TSelectedFields>,
219200
): Promise<Connection<TEntity>> {
220-
const pageResult = await this.knexDataManager.loadPageBySQLFragmentAsync(
221-
this.queryContext,
222-
args,
223-
);
224-
201+
const pageResult = await this.knexDataManager.loadPageAsync(this.queryContext, args);
225202
const edges = await Promise.all(
226203
pageResult.edges.map(async (edge) => {
227204
const entityResult = await this.constructionUtils.constructAndAuthorizeEntityAsync(

‎packages/entity-database-adapter-knex/src/PaginationStrategy.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/**
2+
* Search strategy for SQL-based pagination.
3+
*/
4+
export enum PaginationStrategy {
5+
/**
6+
* Standard pagination with ORDER BY. Results are ordered by the specified orderBy fields, with ID field automatically included for stable pagination if not already present.
7+
*/
8+
STANDARD = 'standard',
9+
10+
/**
11+
* Case-insensitive pattern matching search using SQL ILIKE operator.
12+
* Results are ordered by the fields being searched within in the order specified, then by ID for tie-breaking and stable pagination.
13+
*/
14+
ILIKE_SEARCH = 'ilike-search',
15+
16+
/**
17+
* Similarity search using PostgreSQL trigram similarity. Results are ordered by exact match priority, then by similarity score, then by specified extra order by fields if provided, then by ID for tie-breaking and stable pagination.
18+
*
19+
* Performance considerations:
20+
* - Trigram search can be significantly slower than ILIKE search, especially on large datasets without appropriate indexes.
21+
* - Consider using ILIKE search for smaller datasets or when exact substring matching is sufficient
22+
* - For larger datasets, ensure proper indexing or consider dedicated full-text search solutions.
23+
* - For optimal performance, create GIN or GIST indexes on searchable columns:
24+
* ```sql
25+
* CREATE EXTENSION IF NOT EXISTS pg_trgm;
26+
* CREATE INDEX idx_table_field_trigram ON table_name USING gin(field_name gin_trgm_ops);
27+
* -- Or for multiple columns:
28+
* CREATE INDEX idx_table_search ON table_name USING gin((field1 || ' ' || field2) gin_trgm_ops);
29+
* ```
30+
*/
31+
TRIGRAM_SEARCH = 'trigram',
32+
}

‎packages/entity-database-adapter-knex/src/PostgresEntityDatabaseAdapter.ts‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -110,14 +110,19 @@ export class PostgresEntityDatabaseAdapter<
110110
query: Knex.QueryBuilder,
111111
querySelectionModifiers: TableQuerySelectionModifiersWithOrderByRaw,
112112
): Knex.QueryBuilder {
113-
let ret = this.applyQueryModifiersToQuery(query, querySelectionModifiers);
114-
115113
const { orderByRaw } = querySelectionModifiers;
114+
115+
// orderByRaw takes precedence over orderBy - they are mutually exclusive
116116
if (orderByRaw !== undefined) {
117-
ret = ret.orderByRaw(orderByRaw);
117+
// Apply only orderByRaw (offset/limit still applied, but not orderBy)
118+
return this.applyQueryModifiersToQuery(query, {
119+
...querySelectionModifiers,
120+
orderBy: undefined, // Explicitly exclude orderBy when orderByRaw is present
121+
}).orderByRaw(orderByRaw);
122+
} else {
123+
// Apply regular orderBy (and offset/limit)
124+
return this.applyQueryModifiersToQuery(query, querySelectionModifiers);
118125
}
119-
120-
return ret;
121126
}
122127

123128
private applyQueryModifiersToQuery(
@@ -212,14 +217,24 @@ export class PostgresEntityDatabaseAdapter<
212217
sqlFragment: SQLFragment,
213218
querySelectionModifiers: TableQuerySelectionModifiersWithOrderByFragment,
214219
): Promise<object[]> {
220+
// Build the base query with window function for total count
215221
let query = queryInterface
216-
.select()
222+
.select('*')
217223
.from(tableName)
218224
.whereRaw(sqlFragment.sql, sqlFragment.getKnexBindings());
219-
query = this.applyQueryModifiersToQuery(query, querySelectionModifiers);
225+
226+
// Apply order by modifiers
227+
// orderByFragment takes precedence over orderBy - they are mutually exclusive
220228
const { orderByFragment } = querySelectionModifiers;
221229
if (orderByFragment !== undefined) {
222-
query = query.orderByRaw(orderByFragment.sql, orderByFragment.getKnexBindings());
230+
// Apply only orderByFragment (offset/limit still applied, but not orderBy)
231+
query = this.applyQueryModifiersToQuery(query, {
232+
...querySelectionModifiers,
233+
orderBy: undefined, // Explicitly exclude orderBy when orderByFragment is present
234+
}).orderByRaw(orderByFragment.sql, orderByFragment.getKnexBindings());
235+
} else {
236+
// Apply regular orderBy (and offset/limit)
237+
query = this.applyQueryModifiersToQuery(query, querySelectionModifiers);
223238
}
224239
return await wrapNativePostgresCallAsync(() => query);
225240
}

0 commit comments

Comments
 (0)