feat(orm): implement json filters (#472)

Author: ymc9

TODO.md

@@ -86,7 +86,7 @@
         - [x] Custom table name
         - [x] Custom field name
         - [x] Global omit
-        - [ ] DbNull vs JsonNull
+        - [x] DbNull vs JsonNull
         - [ ] Migrate to tsdown
         - [x] @default validation
     - [x] Benchmark

packages/orm/src/client/crud-types.ts

@@ -458,8 +458,16 @@ export type BooleanFilter<Nullable extends boolean, WithAggregations extends boo
           : {}));
 
 export type JsonFilter = {
+    path?: string[];
     equals?: JsonValue | JsonNullValues;
     not?: JsonValue | JsonNullValues;
+    string_contains?: string;
+    string_starts_with?: string;
+    string_ends_with?: string;
+    mode?: 'default' | 'insensitive';
+    array_contains?: JsonValue;
+    array_starts_with?: JsonValue;
+    array_ends_with?: JsonValue;
 };
 
 // TODO: extra typedef filtering

packages/orm/src/client/crud/dialects/base-dialect.ts

@@ -524,16 +524,57 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
     private buildJsonFilter(lhs: Expression<any>, payload: any): any {
         const clauses: Expression<SqlBool>[] = [];
         invariant(payload && typeof payload === 'object', 'Json filter payload must be an object');
+
+        const path = payload.path && Array.isArray(payload.path) ? payload.path : [];
+        const receiver = this.buildJsonPathSelection(lhs, path, 'json');
+        const stringReceiver = this.buildJsonPathSelection(lhs, path, 'string');
+
+        const mode = payload.mode ?? 'default';
+        invariant(mode === 'default' || mode === 'insensitive', 'Invalid JSON filter mode');
+
         for (const [key, value] of Object.entries(payload)) {
             switch (key) {
                 case 'equals': {
-                    clauses.push(this.buildJsonValueFilterClause(lhs, value));
+                    clauses.push(this.buildJsonValueFilterClause(receiver, value));
                     break;
                 }
                 case 'not': {
-                    clauses.push(this.eb.not(this.buildJsonValueFilterClause(lhs, value)));
+                    clauses.push(this.eb.not(this.buildJsonValueFilterClause(receiver, value)));
+                    break;
+                }
+                case 'string_contains': {
+                    invariant(typeof value === 'string', 'string_contains value must be a string');
+                    clauses.push(this.buildJsonStringFilter(stringReceiver, key, value, mode));
+                    break;
+                }
+                case 'string_starts_with': {
+                    invariant(typeof value === 'string', 'string_starts_with value must be a string');
+                    clauses.push(this.buildJsonStringFilter(stringReceiver, key, value, mode));
+                    break;
+                }
+                case 'string_ends_with': {
+                    invariant(typeof value === 'string', 'string_ends_with value must be a string');
+                    clauses.push(this.buildJsonStringFilter(stringReceiver, key, value, mode));
+                    break;
+                }
+                case 'array_contains': {
+                    clauses.push(this.buildJsonArrayFilter(receiver, key, value));
+                    break;
+                }
+                case 'array_starts_with': {
+                    clauses.push(this.buildJsonArrayFilter(receiver, key, value));
+                    break;
+                }
+                case 'array_ends_with': {
+                    clauses.push(this.buildJsonArrayFilter(receiver, key, value));
                     break;
                 }
+                case 'path':
+                case 'mode':
+                    // already handled
+                    break;
+                default:
+                    invariant(false, `Invalid JSON filter key: ${key}`);
             }
         }
         return this.and(...clauses);
@@ -552,6 +593,24 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
         }
     }
 
+    private buildJsonStringFilter(
+        lhs: Expression<any>,
+        operation: 'string_contains' | 'string_starts_with' | 'string_ends_with',
+        value: string,
+        mode: 'default' | 'insensitive',
+    ) {
+        // build LIKE pattern based on operation
+        const pattern = match(operation)
+            .with('string_contains', () => `%${value}%`)
+            .with('string_starts_with', () => `${value}%`)
+            .with('string_ends_with', () => `%${value}`)
+            .exhaustive();
+
+        // use appropriate operator based on database capabilities
+        const { supportsILike } = this.getStringCasingBehavior();
+        return this.eb(lhs, mode === 'insensitive' && supportsILike ? 'ilike' : 'like', sql.val(pattern));
+    }
+
     private buildLiteralFilter(lhs: Expression<any>, type: BuiltinType, rhs: unknown) {
         return this.eb(lhs, '=', rhs !== null && rhs !== undefined ? this.transformPrimitive(rhs, type, false) : rhs);
     }
@@ -1245,5 +1304,24 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
      */
     abstract getStringCasingBehavior(): { supportsILike: boolean; likeCaseSensitive: boolean };
 
+    /**
+     * Builds a JSON path selection expression.
+     * @param asType 'string' | 'json', when 'string', the result is stripped with text quotes if it's a string
+     */
+    protected abstract buildJsonPathSelection(
+        receiver: Expression<any>,
+        path: string[],
+        asType: 'string' | 'json',
+    ): Expression<any>;
+
+    /**
+     * Builds a JSON array filter expression.
+     */
+    protected abstract buildJsonArrayFilter(
+        lhs: Expression<any>,
+        operation: 'array_contains' | 'array_starts_with' | 'array_ends_with',
+        value: unknown,
+    ): Expression<SqlBool>;
+
     // #endregion
 }

packages/orm/src/client/crud/dialects/postgresql.ts

@@ -7,6 +7,7 @@ import {
     type ExpressionWrapper,
     type RawBuilder,
     type SelectQueryBuilder,
+    type SqlBool,
 } from 'kysely';
 import { match } from 'ts-pattern';
 import z from 'zod';
@@ -453,6 +454,50 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDiale
         }
     }
 
+    protected override buildJsonPathSelection(receiver: Expression<any>, path: string[], asType: 'string' | 'json') {
+        if (path.length > 0) {
+            const pathValues = path.map((p: string) => this.eb.val(p));
+            if (asType === 'string') {
+                // use `jsonb_extract_path_text` to get string values without quotes
+                return this.eb.fn('jsonb_extract_path_text', [receiver, ...pathValues]);
+            } else {
+                return this.eb.fn('jsonb_extract_path', [receiver, ...pathValues]);
+            }
+        } else {
+            // if we're selecting the JSON root, we'll have to resort to `trim` when selecting as string
+            // to remove the quotes
+            if (asType === 'string') {
+                return sql`trim(both '"' from ${receiver}::text)`;
+            } else {
+                return receiver;
+            }
+        }
+    }
+
+    protected override buildJsonArrayFilter(
+        lhs: Expression<any>,
+        operation: 'array_contains' | 'array_starts_with' | 'array_ends_with',
+        value: unknown,
+    ) {
+        return match(operation)
+            .with('array_contains', () => sql<SqlBool>`${lhs} @> ${sql.val(JSON.stringify([value]))}::jsonb`)
+            .with('array_starts_with', () =>
+                this.eb(
+                    this.eb.fn('jsonb_extract_path', [lhs, this.eb.val('0')]),
+                    '=',
+                    this.transformPrimitive(value, 'Json', false),
+                ),
+            )
+            .with('array_ends_with', () =>
+                this.eb(
+                    this.eb.fn('jsonb_extract_path', [lhs, sql`(jsonb_array_length(${lhs}) - 1)::text`]),
+                    '=',
+                    this.transformPrimitive(value, 'Json', false),
+                ),
+            )
+            .exhaustive();
+    }
+
     override get supportInsertWithDefault() {
         return true;
     }

packages/orm/src/client/crud/dialects/sqlite.ts

@@ -359,6 +359,47 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
         );
     }
 
+    protected override buildJsonPathSelection(
+        receiver: Expression<any>,
+        path: string[],
+        _asType: 'string' | 'json',
+    ): Expression<any> {
+        if (path.length === 0) {
+            return receiver;
+        }
+
+        // build a JSON path from the path segments
+        // array indices should use bracket notation: $.a[0].b instead of $.a.0.b
+        const jsonPath =
+            '$' +
+            path
+                .map((p) => {
+                    // check if the segment is a numeric array index
+                    if (/^\d+$/.test(p)) {
+                        return `[${p}]`;
+                    }
+                    return `.${p}`;
+                })
+                .join('');
+        return this.eb.fn('json_extract', [receiver, this.eb.val(jsonPath)]);
+    }
+
+    protected override buildJsonArrayFilter(
+        lhs: Expression<any>,
+        operation: 'array_contains' | 'array_starts_with' | 'array_ends_with',
+        value: unknown,
+    ) {
+        return match(operation)
+            .with('array_contains', () => sql<any>`EXISTS (SELECT 1 FROM json_each(${lhs}) WHERE value = ${value})`)
+            .with('array_starts_with', () =>
+                this.eb(this.eb.fn('json_extract', [lhs, this.eb.val('$[0]')]), '=', value),
+            )
+            .with('array_ends_with', () =>
+                this.eb(sql`json_extract(${lhs}, '$[' || (json_array_length(${lhs}) - 1) || ']')`, '=', value),
+            )
+            .exhaustive();
+    }
+
     override get supportsUpdateWithLimit() {
         return false;
     }

packages/orm/src/client/crud/validator/index.ts

@@ -608,9 +608,17 @@ export class InputValidator<Schema extends SchemaDef> {
 
     private makeJsonFilterSchema(optional: boolean) {
         const valueSchema = this.makeJsonValueSchema(optional, true);
-        return z.object({
+        return z.strictObject({
+            path: z.string().array().optional(),
             equals: valueSchema.optional(),
             not: valueSchema.optional(),
+            string_contains: z.string().optional(),
+            string_starts_with: z.string().optional(),
+            string_ends_with: z.string().optional(),
+            mode: this.makeStringModeSchema().optional(),
+            array_contains: valueSchema.optional(),
+            array_starts_with: valueSchema.optional(),
+            array_ends_with: valueSchema.optional(),
         });
     }
 

packages/testtools/src/client.ts

@@ -157,6 +157,7 @@ export async function createTestClient(
 
     if (options?.debug) {
         console.log(`Work directory: ${workDir}`);
+        console.log(`Database name: ${dbName}`);
         _options.log = testLogger;
     }