Skip to content

Commit 16b0067

Browse files
wschurmanwschurman
committed
feat: Add ilike and trigram similarity search to pagination
1 parent 50a20a7 commit 16b0067

3 files changed

Lines changed: 293 additions & 4 deletions

File tree

‎packages/entity-database-adapter-knex/src/AuthorizationResultBasedKnexEntityLoader.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,39 @@ export interface EntityLoaderQuerySelectionModifiersWithOrderByFragment<
7676
orderByFragment?: SQLFragment;
7777
}
7878

79+
/**
80+
* Search strategy for SQL-based pagination.
81+
*/
82+
export enum SearchStrategy {
83+
ILIKE = 'ilike',
84+
TRIGRAM = 'trigram',
85+
}
86+
87+
interface SearchSpecificationBase<TFields extends Record<string, any>> {
88+
term: string;
89+
fields: (keyof TFields)[];
90+
}
91+
92+
interface ILikeSearchSpecification<
93+
TFields extends Record<string, any>,
94+
> extends SearchSpecificationBase<TFields> {
95+
strategy: SearchStrategy.ILIKE;
96+
}
97+
98+
interface TrigramSearchSpecification<
99+
TFields extends Record<string, any>,
100+
> extends SearchSpecificationBase<TFields> {
101+
strategy: SearchStrategy.TRIGRAM;
102+
threshold: number;
103+
}
104+
105+
/**
106+
* Search specification for SQL-based pagination.
107+
*/
108+
export type SearchSpecification<TFields extends Record<string, any>> =
109+
| ILikeSearchSpecification<TFields>
110+
| TrigramSearchSpecification<TFields>;
111+
79112
/**
80113
* Base pagination arguments
81114
*/
@@ -104,6 +137,12 @@ interface EntityLoaderBasePaginationArgs<
104137
* The total count is available in the connection's totalCount field.
105138
*/
106139
includeTotal?: boolean;
140+
141+
/**
142+
* Search specification for filtering entities by text search.
143+
* Supports ILIKE pattern matching or PostgreSQL trigram similarity search.
144+
*/
145+
search?: SearchSpecification<TFields>;
107146
}
108147

109148
/**

‎packages/entity-database-adapter-knex/src/__integration-tests__/PostgresEntityIntegration-test.ts‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { knex, Knex } from 'knex';
1010
import nullthrows from 'nullthrows';
1111
import { setTimeout } from 'timers/promises';
1212

13+
import { SearchStrategy } from '../AuthorizationResultBasedKnexEntityLoader';
1314
import { OrderByOrdering } from '../BasePostgresEntityDatabaseAdapter';
1415
import { raw, sql, SQLFragment, SQLFragmentHelpers } from '../SQLOperator';
1516
import { PostgresTestEntity } from '../__testfixtures__/PostgresTestEntity';
@@ -1862,5 +1863,175 @@ describe('postgres entity integration', () => {
18621863
expect(pageDefaultNoTotal.edges).toHaveLength(3);
18631864
expect(pageDefaultNoTotal.totalCount).toBeUndefined();
18641865
});
1866+
1867+
it('supports search with ILIKE strategy', async () => {
1868+
const vc = new ViewerContext(createKnexIntegrationTestEntityCompanionProvider(knexInstance));
1869+
1870+
await PostgresTestEntity.dropPostgresTableAsync(knexInstance);
1871+
await PostgresTestEntity.createOrTruncatePostgresTableAsync(knexInstance);
1872+
1873+
// Create test data with searchable names
1874+
const names = ['Alice Johnson', 'Bob Smith', 'Charlie Brown', 'David Smith', 'Eve Johnson', 'Frank Miller'];
1875+
for (let i = 0; i < names.length; i++) {
1876+
await PostgresTestEntity.creator(vc)
1877+
.setField('name', names[i]!)
1878+
.setField('hasACat', i % 2 === 0)
1879+
.createAsync();
1880+
}
1881+
1882+
// Search for names containing "Johnson"
1883+
const searchResults = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1884+
first: 10,
1885+
search: {
1886+
strategy: SearchStrategy.ILIKE,
1887+
term: 'Johnson',
1888+
fields: ['name'],
1889+
},
1890+
orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
1891+
});
1892+
1893+
expect(searchResults.edges).toHaveLength(2);
1894+
expect(searchResults.edges[0]?.node.getField('name')).toBe('Alice Johnson');
1895+
expect(searchResults.edges[1]?.node.getField('name')).toBe('Eve Johnson');
1896+
1897+
// Search for names containing "Smith" with pagination
1898+
const smithPage1 = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1899+
first: 1,
1900+
search: {
1901+
strategy: SearchStrategy.ILIKE,
1902+
term: 'Smith',
1903+
fields: ['name'],
1904+
},
1905+
orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
1906+
});
1907+
1908+
expect(smithPage1.edges).toHaveLength(1);
1909+
expect(smithPage1.edges[0]?.node.getField('name')).toBe('Bob Smith');
1910+
expect(smithPage1.pageInfo.hasNextPage).toBe(true);
1911+
1912+
// Get next page
1913+
const smithPage2 = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1914+
first: 1,
1915+
after: smithPage1.pageInfo.endCursor!,
1916+
search: {
1917+
strategy: SearchStrategy.ILIKE,
1918+
term: 'Smith',
1919+
fields: ['name'],
1920+
},
1921+
orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
1922+
});
1923+
1924+
expect(smithPage2.edges).toHaveLength(1);
1925+
expect(smithPage2.edges[0]?.node.getField('name')).toBe('David Smith');
1926+
expect(smithPage2.pageInfo.hasNextPage).toBe(false);
1927+
1928+
// Test partial match (case insensitive)
1929+
const partialMatch = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1930+
first: 10,
1931+
search: {
1932+
strategy: SearchStrategy.ILIKE,
1933+
term: 'john',
1934+
fields: ['name'],
1935+
},
1936+
orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
1937+
});
1938+
1939+
expect(partialMatch.edges).toHaveLength(2);
1940+
expect(partialMatch.edges[0]?.node.getField('name')).toBe('Alice Johnson');
1941+
expect(partialMatch.edges[1]?.node.getField('name')).toBe('Eve Johnson');
1942+
1943+
// Test search with WHERE clause
1944+
const combinedFilter = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1945+
first: 10,
1946+
where: sql`has_a_cat = ${true}`,
1947+
search: {
1948+
strategy: SearchStrategy.ILIKE,
1949+
term: 'Johnson',
1950+
fields: ['name'],
1951+
},
1952+
orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
1953+
});
1954+
1955+
// Both Alice Johnson (index 0) and Eve Johnson (index 4) have cats
1956+
expect(combinedFilter.edges).toHaveLength(2);
1957+
expect(combinedFilter.edges[0]?.node.getField('name')).toBe('Alice Johnson');
1958+
expect(combinedFilter.edges[0]?.node.getField('hasACat')).toBe(true);
1959+
expect(combinedFilter.edges[1]?.node.getField('name')).toBe('Eve Johnson');
1960+
expect(combinedFilter.edges[1]?.node.getField('hasACat')).toBe(true);
1961+
1962+
// Test search with includeTotal
1963+
const withTotal = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1964+
first: 1,
1965+
search: {
1966+
strategy: SearchStrategy.ILIKE,
1967+
term: 'Smith',
1968+
fields: ['name'],
1969+
},
1970+
orderBy: [{ fieldName: 'name', order: OrderByOrdering.ASCENDING }],
1971+
includeTotal: true,
1972+
});
1973+
1974+
expect(withTotal.edges).toHaveLength(1);
1975+
expect(withTotal.totalCount).toBe(2); // Bob Smith and David Smith
1976+
});
1977+
1978+
it('supports trigram similarity search', async () => {
1979+
const vc = new ViewerContext(createKnexIntegrationTestEntityCompanionProvider(knexInstance));
1980+
1981+
await PostgresTestEntity.dropPostgresTableAsync(knexInstance);
1982+
await PostgresTestEntity.createOrTruncatePostgresTableAsync(knexInstance);
1983+
1984+
// Enable pg_trgm extension for trigram similarity
1985+
await knexInstance.raw('CREATE EXTENSION IF NOT EXISTS pg_trgm');
1986+
1987+
// Create test data with similar names
1988+
const names = ['Johnson', 'Jonson', 'Johnsen', 'Smith', 'Smyth', 'Schmidt'];
1989+
for (let i = 0; i < names.length; i++) {
1990+
await PostgresTestEntity.creator(vc)
1991+
.setField('name', names[i]!)
1992+
.setField('hasACat', i < 3) // First 3 have cats
1993+
.createAsync();
1994+
}
1995+
1996+
// Search for similar names to "Johnson" using trigram
1997+
const trigramSearch = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
1998+
first: 10,
1999+
search: {
2000+
strategy: SearchStrategy.TRIGRAM,
2001+
term: 'Johnson',
2002+
fields: ['name'],
2003+
threshold: 0.3, // Similarity threshold
2004+
},
2005+
});
2006+
2007+
// Should find exact match and similar names, ordered by relevance
2008+
expect(trigramSearch.edges.length).toBeGreaterThan(0);
2009+
// Exact match should come first due to ILIKE matching
2010+
expect(trigramSearch.edges[0]?.node.getField('name')).toBe('Johnson');
2011+
2012+
// The similar names (Jonson, Johnsen) should also be included
2013+
const foundNames = trigramSearch.edges.map(e => e.node.getField('name'));
2014+
expect(foundNames).toContain('Jonson');
2015+
expect(foundNames).toContain('Johnsen');
2016+
2017+
// Test combining with WHERE clause
2018+
const filteredTrigram = await PostgresTestEntity.knexLoader(vc).loadPageBySQLAsync({
2019+
first: 10,
2020+
where: sql`has_a_cat = ${true}`,
2021+
search: {
2022+
strategy: SearchStrategy.TRIGRAM,
2023+
term: 'Johnson',
2024+
fields: ['name'],
2025+
threshold: 0.3,
2026+
},
2027+
});
2028+
2029+
// Only the Johnson-like names with cats
2030+
expect(filteredTrigram.edges.length).toBeGreaterThan(0);
2031+
expect(filteredTrigram.edges.length).toBeLessThanOrEqual(3);
2032+
filteredTrigram.edges.forEach(edge => {
2033+
expect(edge.node.getField('hasACat')).toBe(true);
2034+
});
2035+
});
18652036
});
18662037
});

‎packages/entity-database-adapter-knex/src/internal/EntityKnexDataManager.ts‎

Lines changed: 83 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
PostgresQuerySelectionModifiersWithOrderByFragment,
1919
PostgresQuerySelectionModifiersWithOrderByRaw,
2020
} from '../BasePostgresEntityDatabaseAdapter';
21+
import { SearchStrategy, type SearchSpecification } from '../AuthorizationResultBasedKnexEntityLoader';
2122
import { SQLFragment, identifier, raw, sql } from '../SQLOperator';
2223

2324
/**
@@ -28,6 +29,7 @@ interface BasePaginationArgs<TFields extends Record<string, any>> {
2829
orderBy?: PostgresOrderByClause<TFields>[];
2930
cursorFields?: readonly (keyof TFields)[];
3031
includeTotal?: boolean;
32+
search?: SearchSpecification<TFields>;
3133
}
3234

3335
/**
@@ -210,7 +212,7 @@ export class EntityKnexDataManager<
210212
}
211213

212214
const isForward = 'first' in args;
213-
const { where, orderBy, cursorFields, includeTotal } = args;
215+
const { where, orderBy, cursorFields, includeTotal, search } = args;
214216

215217
let limit: number;
216218
let cursor: string | undefined;
@@ -234,9 +236,27 @@ export class EntityKnexDataManager<
234236
// Decode cursor
235237
const decodedCursor = cursor ? this.decodeCursor(cursor, finalCursorFields) : null;
236238

239+
// Build search conditions if search is provided
240+
let searchWhere: SQLFragment | undefined;
241+
let searchOrderByFragment: SQLFragment | undefined;
242+
if (search) {
243+
const searchConditions = this.buildSearchConditionAndOrderBy(search);
244+
searchWhere = searchConditions.searchWhere;
245+
searchOrderByFragment = searchConditions.searchOrderByFragment;
246+
}
247+
248+
// Combine WHERE conditions: base where + search where + cursor condition
249+
const combinedWhere = [where, searchWhere].filter(Boolean).reduce<SQLFragment | undefined>(
250+
(acc, condition) => {
251+
if (!acc) return condition;
252+
return sql`(${acc}) AND (${condition})`;
253+
},
254+
undefined,
255+
);
256+
237257
// Build WHERE clause with cursor condition for keyset pagination
238258
const whereClause = this.buildWhereClause({
239-
...(where && { where }),
259+
...(combinedWhere && { where: combinedWhere }),
240260
cursorFields: finalCursorFields,
241261
decodedCursor,
242262
direction: isForward ? 'forward' : 'backward',
@@ -262,15 +282,21 @@ export class EntityKnexDataManager<
262282
queryContext,
263283
)(
264284
this.databaseAdapter.fetchManyBySQLFragmentAsync(queryContext, whereClause, {
265-
orderBy: finalOrderByClauses,
285+
// Use orderByFragment if search provides one, otherwise use regular orderBy
286+
...(searchOrderByFragment
287+
? { orderByFragment: searchOrderByFragment }
288+
: { orderBy: finalOrderByClauses }),
266289
limit: limit + 1,
267290
}),
268291
);
269292

270293
const [fieldObjects, totalCount] = await Promise.all([
271294
fetchPromise,
272295
includeTotal
273-
? this.databaseAdapter.fetchCountBySQLFragmentAsync(queryContext, where ?? sql`1 = 1`)
296+
? this.databaseAdapter.fetchCountBySQLFragmentAsync(
297+
queryContext,
298+
combinedWhere ?? sql`1 = 1`,
299+
)
274300
: Promise.resolve(undefined),
275301
]);
276302

@@ -400,4 +426,57 @@ export class EntityKnexDataManager<
400426
)})
401427
`;
402428
}
429+
430+
private buildSearchConditionAndOrderBy(
431+
search: SearchSpecification<TFields>,
432+
): { searchWhere: SQLFragment; searchOrderByFragment: SQLFragment | undefined } {
433+
switch (search.strategy) {
434+
case SearchStrategy.ILIKE: {
435+
// Simple ILIKE search
436+
const conditions = search.fields.map((field) => {
437+
const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
438+
return sql`${identifier(dbField)} ILIKE ${'%' + search.term + '%'}`;
439+
});
440+
return {
441+
searchWhere: conditions.length > 0 ? SQLFragment.join(conditions, ' OR ') : sql`1 = 0`,
442+
searchOrderByFragment: undefined,
443+
};
444+
}
445+
446+
case SearchStrategy.TRIGRAM: {
447+
// PostgreSQL trigram similarity
448+
const conditions = search.fields.map((field) => {
449+
const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
450+
return sql`similarity(${identifier(dbField)}, ${search.term}) > ${search.threshold}`;
451+
});
452+
const ilikeConditions = search.fields.map((field) => {
453+
const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
454+
return sql`${identifier(dbField)} ILIKE ${'%' + search.term + '%'}`;
455+
});
456+
// Combine exact matches (ILIKE) with similarity
457+
const allConditions = [...ilikeConditions, ...conditions];
458+
const idField = getDatabaseFieldForEntityField(this.entityConfiguration, this.entityConfiguration.idField);
459+
return {
460+
searchWhere: SQLFragment.join(allConditions, ' OR '),
461+
// For trigram search, order by relevance
462+
// 1. Exact matches first (ILIKE)
463+
// 2. Then by similarity score
464+
// 3. Then by ID field
465+
searchOrderByFragment: sql`CASE WHEN ${SQLFragment.join(
466+
search.fields.map((field) => {
467+
const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
468+
return sql`${identifier(dbField)} ILIKE ${'%' + search.term + '%'}`;
469+
}),
470+
' OR ',
471+
)} THEN 0 ELSE 1 END, GREATEST(${SQLFragment.join(
472+
search.fields.map((field) => {
473+
const dbField = getDatabaseFieldForEntityField(this.entityConfiguration, field);
474+
return sql`similarity(${identifier(dbField)}, ${search.term})`;
475+
}),
476+
', ',
477+
)}) DESC, ${identifier(idField)} DESC`,
478+
};
479+
}
480+
}
481+
}
403482
}

0 commit comments

Comments
 (0)