Skip to content

perf(orm): improve post query data processing performance #2485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ymc9 merged 1 commit into from
Mar 15, 2026
Merged

perf(orm): improve post query data processing performance #2485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 44 additions & 31 deletions packages/orm/src/client/crud/dialects/mysql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import {
type SelectQueryBuilder,
type SqlBool,
} from 'kysely';
import { match } from 'ts-pattern';
import { AnyNullClass, DbNullClass, JsonNullClass } from '../../../common-types';
import type { BuiltinType, FieldDef, SchemaDef } from '../../../schema';
import type { SortOrder } from '../../crud-types';
Expand Down Expand Up @@ -93,9 +92,10 @@ export class MySqlCrudDialect<Schema extends SchemaDef> extends LateralJoinDiale
throw createNotSupportedError(`MySQL does not support array literals`);
}
} else {
return match(type)
.with('Boolean', () => (value ? 1 : 0)) // MySQL uses 1/0 for boolean like SQLite
.with('DateTime', () => {
switch (type) {
case 'Boolean':
return value ? 1 : 0;
case 'DateTime':
// MySQL DATETIME format: 'YYYY-MM-DD HH:MM:SS.mmm'
if (value instanceof Date) {
// force UTC
Expand All @@ -106,29 +106,37 @@ export class MySqlCrudDialect<Schema extends SchemaDef> extends LateralJoinDiale
} else {
return value;
}
})
.with('Decimal', () => (value !== null ? value.toString() : value))
.with('Json', () => {
case 'Decimal':
return value !== null ? value.toString() : value;
case 'Json':
return this.eb.cast(this.eb.val(JSON.stringify(value)), 'json');
})
.with('Bytes', () =>
Buffer.isBuffer(value) ? value : value instanceof Uint8Array ? Buffer.from(value) : value,
)
.otherwise(() => value);
case 'Bytes':
return Buffer.isBuffer(value) ? value : value instanceof Uint8Array ? Buffer.from(value) : value;
default:
return value;
}
}
}

override transformOutput(value: unknown, type: BuiltinType, array: boolean) {
if (value === null || value === undefined) {
return value;
}
return match(type)
.with('Boolean', () => this.transformOutputBoolean(value))
.with('DateTime', () => this.transformOutputDate(value))
.with('Bytes', () => this.transformOutputBytes(value))
.with('BigInt', () => this.transformOutputBigInt(value))
.with('Decimal', () => this.transformDecimal(value))
.otherwise(() => super.transformOutput(value, type, array));

switch (type) {
case 'Boolean':
return this.transformOutputBoolean(value);
case 'DateTime':
return this.transformOutputDate(value);
case 'Bytes':
return this.transformOutputBytes(value);
case 'BigInt':
return this.transformOutputBigInt(value);
case 'Decimal':
return this.transformDecimal(value);
default:
return super.transformOutput(value, type, array);
}
}

private transformOutputBoolean(value: unknown) {
Expand Down Expand Up @@ -282,26 +290,31 @@ export class MySqlCrudDialect<Schema extends SchemaDef> extends LateralJoinDiale
operation: 'array_contains' | 'array_starts_with' | 'array_ends_with',
value: unknown,
) {
return match(operation)
.with('array_contains', () => {
switch (operation) {
case 'array_contains': {
const v = Array.isArray(value) ? value : [value];
return sql<SqlBool>`JSON_CONTAINS(${lhs}, ${sql.val(JSON.stringify(v))})`;
})
.with('array_starts_with', () =>
this.eb(
}

case 'array_starts_with': {
return this.eb(
this.eb.fn('JSON_EXTRACT', [lhs, this.eb.val('$[0]')]),
'=',
this.transformInput(value, 'Json', false),
),
)
.with('array_ends_with', () =>
this.eb(
);
}

case 'array_ends_with': {
return this.eb(
sql`JSON_EXTRACT(${lhs}, CONCAT('$[', JSON_LENGTH(${lhs}) - 1, ']'))`,
'=',
this.transformInput(value, 'Json', false),
),
)
.exhaustive();
);
}

default:
throw createInvalidInputError(`Unsupported array filter operation: ${operation}`);
}
}

protected override buildJsonArrayExistsPredicate(
Expand Down
96 changes: 50 additions & 46 deletions packages/orm/src/client/crud/dialects/postgresql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import {
type SqlBool,
} from 'kysely';
import { parse as parsePostgresArray } from 'postgres-array';
import { match } from 'ts-pattern';
import { AnyNullClass, DbNullClass, JsonNullClass } from '../../../common-types';
import type { BuiltinType, FieldDef, SchemaDef } from '../../../schema';
import type { SortOrder } from '../../crud-types';
Expand All @@ -21,6 +20,18 @@ import { LateralJoinDialectBase } from './lateral-join-dialect-base';
export class PostgresCrudDialect<Schema extends SchemaDef> extends LateralJoinDialectBase<Schema> {
private static typeParserOverrideApplied = false;

private readonly zmodelToSqlTypeMap: Record<string, string> = {
String: 'text',
Boolean: 'boolean',
Int: 'integer',
BigInt: 'bigint',
Float: 'double precision',
Decimal: 'decimal',
DateTime: 'timestamp',
Bytes: 'bytea',
Json: 'jsonb',
};

constructor(schema: Schema, options: ClientOptions<Schema>) {
super(schema, options);
this.overrideTypeParsers();
Expand Down Expand Up @@ -144,16 +155,16 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends LateralJoinDi
return value.map((v) => this.transformInput(v, type, false));
}
} else {
return match(type)
.with('DateTime', () =>
value instanceof Date
switch (type) {
case 'DateTime':
return value instanceof Date
? value.toISOString()
: typeof value === 'string'
? new Date(value).toISOString()
: value,
)
.with('Decimal', () => (value !== null ? value.toString() : value))
.with('Json', () => {
: value;
case 'Decimal':
return value !== null ? value.toString() : value;
case 'Json':
if (
value === null ||
typeof value === 'string' ||
Expand All @@ -165,25 +176,33 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends LateralJoinDi
} else {
return value;
}
})
.otherwise(() => value);
default:
return value;
}
}
}

override transformOutput(value: unknown, type: BuiltinType, array: boolean) {
if (value === null || value === undefined) {
return value;
}
return match(type)
.with('DateTime', () => this.transformOutputDate(value))
.with('Bytes', () => this.transformOutputBytes(value))
.with('BigInt', () => this.transformOutputBigInt(value))
.with('Decimal', () => this.transformDecimal(value))
.when(
(type) => isEnum(this.schema, type),
() => this.transformOutputEnum(value, array),
)
.otherwise(() => super.transformOutput(value, type, array));

switch (type) {
case 'DateTime':
return this.transformOutputDate(value);
case 'Bytes':
return this.transformOutputBytes(value);
case 'BigInt':
return this.transformOutputBigInt(value);
case 'Decimal':
return this.transformDecimal(value);
default:
if (isEnum(this.schema, type)) {
return this.transformOutputEnum(value, array);
} else {
return super.transformOutput(value, type, array);
}
}
}

private transformOutputBigInt(value: unknown) {
Expand Down Expand Up @@ -339,26 +358,24 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends LateralJoinDi
operation: 'array_contains' | 'array_starts_with' | 'array_ends_with',
value: unknown,
) {
return match(operation)
.with('array_contains', () => {
switch (operation) {
case 'array_contains': {
const v = Array.isArray(value) ? value : [value];
return sql<SqlBool>`${lhs} @> ${sql.val(JSON.stringify(v))}::jsonb`;
})
.with('array_starts_with', () =>
this.eb(
}
case 'array_starts_with':
return this.eb(
this.eb.fn('jsonb_extract_path', [lhs, this.eb.val('0')]),
'=',
this.transformInput(value, 'Json', false),
),
)
.with('array_ends_with', () =>
this.eb(
);
case 'array_ends_with':
return this.eb(
this.eb.fn('jsonb_extract_path', [lhs, sql`(jsonb_array_length(${lhs}) - 1)::text`]),
'=',
this.transformInput(value, 'Json', false),
),
)
.exhaustive();
);
}
}

protected override buildJsonArrayExistsPredicate(
Expand All @@ -378,20 +395,7 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends LateralJoinDi
// reduce enum to text for type compatibility
return 'text';
} else {
return (
match(zmodelType)
.with('String', () => 'text')
.with('Boolean', () => 'boolean')
.with('Int', () => 'integer')
.with('BigInt', () => 'bigint')
.with('Float', () => 'double precision')
.with('Decimal', () => 'decimal')
.with('DateTime', () => 'timestamp')
.with('Bytes', () => 'bytea')
.with('Json', () => 'jsonb')
// fallback to text
.otherwise(() => 'text')
);
return this.zmodelToSqlTypeMap[zmodelType] ?? 'text';
}
}

Expand Down
64 changes: 36 additions & 28 deletions packages/orm/src/client/crud/dialects/sqlite.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
type SelectQueryBuilder,
type SqlBool,
} from 'kysely';
import { match } from 'ts-pattern';
import { AnyNullClass, DbNullClass, JsonNullClass } from '../../../common-types';
import type { BuiltinType, FieldDef, GetModels, SchemaDef } from '../../../schema';
import { DELEGATE_JOINED_FIELD_PREFIX } from '../../constants';
Expand Down Expand Up @@ -90,18 +89,22 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
if (Array.isArray(value)) {
return value.map((v) => this.transformInput(v, type, false));
} else {
return match(type)
.with('Boolean', () => (value ? 1 : 0))
.with('DateTime', () =>
value instanceof Date
switch (type) {
case 'Boolean':
return value ? 1 : 0;
case 'DateTime':
return value instanceof Date
? value.toISOString()
: typeof value === 'string'
? new Date(value).toISOString()
: value,
)
.with('Decimal', () => (value as Decimal).toString())
.with('Bytes', () => Buffer.from(value as Uint8Array))
.otherwise(() => value);
: value;
case 'Decimal':
return (value as Decimal).toString();
case 'Bytes':
return Buffer.from(value as Uint8Array);
default:
return value;
}
}
}

Expand All @@ -112,14 +115,22 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
// typed JSON field
return this.transformOutputJson(value);
} else {
return match(type)
.with('Boolean', () => this.transformOutputBoolean(value))
.with('DateTime', () => this.transformOutputDate(value))
.with('Bytes', () => this.transformOutputBytes(value))
.with('Decimal', () => this.transformOutputDecimal(value))
.with('BigInt', () => this.transformOutputBigInt(value))
.with('Json', () => this.transformOutputJson(value))
.otherwise(() => super.transformOutput(value, type, array));
switch (type) {
case 'Boolean':
return this.transformOutputBoolean(value);
case 'DateTime':
return this.transformOutputDate(value);
case 'Bytes':
return this.transformOutputBytes(value);
case 'Decimal':
return this.transformOutputDecimal(value);
case 'BigInt':
return this.transformOutputBigInt(value);
case 'Json':
return this.transformOutputJson(value);
default:
return super.transformOutput(value, type, array);
}
}
}

Expand Down Expand Up @@ -415,23 +426,20 @@ export class SqliteCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect
operation: 'array_contains' | 'array_starts_with' | 'array_ends_with',
value: unknown,
) {
return match(operation)
.with('array_contains', () => {
switch (operation) {
case 'array_contains':
if (Array.isArray(value)) {
throw createNotSupportedError(
'SQLite "array_contains" only supports checking for a single value, not an array of values',
);
} else {
return sql<any>`EXISTS (SELECT 1 FROM json_each(${lhs}) WHERE value = ${value})`;
}
})
.with('array_starts_with', () =>
this.eb(this.eb.fn('json_extract', [lhs, this.eb.val('$[0]')]), '=', value),
)
.with('array_ends_with', () =>
this.eb(sql`json_extract(${lhs}, '$[' || (json_array_length(${lhs}) - 1) || ']')`, '=', value),
)
.exhaustive();
case 'array_starts_with':
return this.eb(this.eb.fn('json_extract', [lhs, this.eb.val('$[0]')]), '=', value);
case 'array_ends_with':
return this.eb(sql`json_extract(${lhs}, '$[' || (json_array_length(${lhs}) - 1) || ']')`, '=', value);
}
}

protected override buildJsonArrayExistsPredicate(
Expand Down
Loading
Loading