feat(orm): mysql support (#616)

* WIP(orm): mysql support

* WIP: more progress with fixing tests

* WIP: get all client api tests pass

* WIP: get all tests pass

* fix executor

* add MySQL to CI matrix

* fix sqlite test runs

* fix test

* fix delete readback check

* set mysql container max connections

* fix tests

* fix test

* refactor: extract duplicated mysql/pg code into base class

* address PR comments

* refactor: remove order by duplicated code

* refactor: optimize stripTableReference

* addressing PR comments

* fix tests

Author: ymc9

.github/workflows/build-test.yml

@@ -31,10 +31,23 @@ jobs:
                 ports:
                     - 5432:5432
 
+            mysql:
+                image: mysql:8.4
+                env:
+                    MYSQL_ROOT_PASSWORD: mysql
+                ports:
+                    - 3306:3306
+                # Set health checks to wait until mysql has started
+                options: >-
+                    --health-cmd="mysqladmin ping --silent"
+                    --health-interval=10s
+                    --health-timeout=5s
+                    --health-retries=3
+
         strategy:
             matrix:
                 node-version: [22.x]
-                provider: [sqlite, postgresql]
+                provider: [sqlite, postgresql, mysql]
 
         steps:
             - name: Checkout
@@ -81,5 +94,9 @@ jobs:
             - name: Lint
               run: pnpm run lint
 
+            - name: Set MySQL max_connections
+              run: |
+                  mysql -h 127.0.0.1 -uroot -pmysql -e "SET GLOBAL max_connections=500;"
+
             - name: Test
               run: TEST_DB_PROVIDER=${{ matrix.provider }} pnpm run test

TODO.md

@@ -18,7 +18,6 @@
 - [ ] ZModel
     - [x] Import
     - [ ] View support
-    - [ ] Datasource provider-scoped attributes
 - [ ] ORM
     - [x] Create
         - [x] Input validation
@@ -72,7 +71,7 @@
         - [x] Query builder API
         - [x] Computed fields
         - [x] Plugin
-        - [ ] Custom procedures
+        - [x] Custom procedures
     - [ ] Misc
         - [x] JSDoc for CRUD methods
         - [x] Cache validation schemas
@@ -110,4 +109,4 @@
     - [x] SQLite
     - [x] PostgreSQL
         - [x] Multi-schema
-    - [ ] MySQL
+    - [x] MySQL

package.json

@@ -9,8 +9,9 @@
         "watch": "turbo run watch build",
         "lint": "turbo run lint",
         "test": "turbo run test",
-        "test:all": "pnpm run test:sqlite && pnpm run test:pg",
+        "test:all": "pnpm run test:sqlite && pnpm run test:pg && pnpm run test:mysql",
         "test:pg": "TEST_DB_PROVIDER=postgresql turbo run test",
+        "test:mysql": "TEST_DB_PROVIDER=mysql turbo run test",
         "test:sqlite": "TEST_DB_PROVIDER=sqlite turbo run test",
         "test:coverage": "vitest run --coverage",
         "format": "prettier --write \"**/*.{ts,tsx,md}\"",

packages/language/src/constants.ts

@@ -1,14 +1,7 @@
 /**
  * Supported db providers
  */
-export const SUPPORTED_PROVIDERS = [
-    'sqlite',
-    'postgresql',
-    // TODO: other providers
-    // 'mysql',
-    // 'sqlserver',
-    // 'cockroachdb',
-];
+export const SUPPORTED_PROVIDERS = ['sqlite', 'postgresql', 'mysql'];
 
 /**
  * All scalar types
@@ -41,3 +34,8 @@ export enum ExpressionContext {
     ValidationRule = 'ValidationRule',
     Index = 'Index',
 }
+
+/**
+ * Database providers that support list field types.
+ */
+export const DB_PROVIDERS_SUPPORTING_LIST_TYPE = ['postgresql'];

packages/language/src/document.ts

@@ -1,3 +1,4 @@
+import { invariant } from '@zenstackhq/common-helpers';
 import {
     isAstNode,
     TextDocument,
@@ -10,10 +11,18 @@ import {
 import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { isDataSource, type Model } from './ast';
-import { STD_LIB_MODULE_NAME } from './constants';
+import { isDataModel, isDataSource, type Model } from './ast';
+import { DB_PROVIDERS_SUPPORTING_LIST_TYPE, STD_LIB_MODULE_NAME } from './constants';
 import { createZModelServices } from './module';
-import { getDataModelAndTypeDefs, getDocument, hasAttribute, resolveImport, resolveTransitiveImports } from './utils';
+import {
+    getAllFields,
+    getDataModelAndTypeDefs,
+    getDocument,
+    getLiteral,
+    hasAttribute,
+    resolveImport,
+    resolveTransitiveImports,
+} from './utils';
 import type { ZModelFormatter } from './zmodel-formatter';
 
 /**
@@ -207,6 +216,24 @@ function validationAfterImportMerge(model: Model) {
     if (authDecls.length > 1) {
         errors.push('Validation error: Multiple `@@auth` declarations are not allowed');
     }
+
+    // check for usages incompatible with the datasource provider
+    const provider = getDataSourceProvider(model);
+    invariant(provider !== undefined, 'Datasource provider should be defined at this point');
+
+    for (const decl of model.declarations.filter(isDataModel)) {
+        const fields = getAllFields(decl, true);
+        for (const field of fields) {
+            if (field.type.array && !isDataModel(field.type.reference?.ref)) {
+                if (!DB_PROVIDERS_SUPPORTING_LIST_TYPE.includes(provider)) {
+                    errors.push(
+                        `Validation error: List type is not supported for "${provider}" provider (model: "${decl.name}", field: "${field.name}")`,
+                    );
+                }
+            }
+        }
+    }
+
     return errors;
 }
 
@@ -226,3 +253,15 @@ export async function formatDocument(content: string) {
     const edits = await formatter.formatDocument(document, { options, textDocument: identifier });
     return TextDocument.applyEdits(document.textDocument, edits);
 }
+
+function getDataSourceProvider(model: Model) {
+    const dataSource = model.declarations.find(isDataSource);
+    if (!dataSource) {
+        return undefined;
+    }
+    const provider = dataSource?.fields.find((f) => f.name === 'provider');
+    if (!provider) {
+        return undefined;
+    }
+    return getLiteral<string>(provider.value);
+}

packages/language/src/validators/datamodel-validator.ts

@@ -5,20 +5,16 @@ import {
     ArrayExpr,
     DataField,
     DataModel,
-    Model,
     ReferenceExpr,
     TypeDef,
     isDataModel,
-    isDataSource,
     isEnum,
-    isModel,
     isStringLiteral,
     isTypeDef,
 } from '../generated/ast';
 import {
     getAllAttributes,
     getAllFields,
-    getLiteral,
     getModelIdFields,
     getModelUniqueFields,
     getUniqueFields,
@@ -105,13 +101,6 @@ export default class DataModelValidator implements AstValidator<DataModel> {
             accept('error', 'Unsupported type argument must be a string literal', { node: field.type.unsupported });
         }
 
-        if (field.type.array && !isDataModel(field.type.reference?.ref)) {
-            const provider = this.getDataSourceProvider(AstUtils.getContainerOfType(field, isModel)!);
-            if (provider === 'sqlite') {
-                accept('error', `List type is not supported for "${provider}" provider.`, { node: field.type });
-            }
-        }
-
         field.attributes.forEach((attr) => validateAttributeApplication(attr, accept));
 
         if (isTypeDef(field.type.reference?.ref)) {
@@ -121,18 +110,6 @@ export default class DataModelValidator implements AstValidator<DataModel> {
         }
     }
 
-    private getDataSourceProvider(model: Model) {
-        const dataSource = model.declarations.find(isDataSource);
-        if (!dataSource) {
-            return undefined;
-        }
-        const provider = dataSource?.fields.find((f) => f.name === 'provider');
-        if (!provider) {
-            return undefined;
-        }
-        return getLiteral<string>(provider.value);
-    }
-
     private validateAttributes(dm: DataModel, accept: ValidationAcceptor) {
         getAllAttributes(dm).forEach((attr) => validateAttributeApplication(attr, accept, dm));
     }

packages/orm/package.json

@@ -56,6 +56,16 @@
                 "default": "./dist/dialects/postgres.cjs"
             }
         },
+        "./dialects/mysql": {
+            "import": {
+                "types": "./dist/dialects/mysql.d.ts",
+                "default": "./dist/dialects/mysql.js"
+            },
+            "require": {
+                "types": "./dist/dialects/mysql.d.cts",
+                "default": "./dist/dialects/mysql.cjs"
+            }
+        },
         "./dialects/sql.js": {
             "import": {
                 "types": "./dist/dialects/sql.js.d.ts",
@@ -100,6 +110,7 @@
     "peerDependencies": {
         "better-sqlite3": "catalog:",
         "pg": "catalog:",
+        "mysql2": "catalog:",
         "sql.js": "catalog:",
         "zod": "catalog:"
     },
@@ -110,6 +121,9 @@
         "pg": {
             "optional": true
         },
+        "mysql2": {
+            "optional": true
+        },
         "sql.js": {
             "optional": true
         }

packages/orm/src/client/client-impl.ts

@@ -31,7 +31,7 @@ import { FindOperationHandler } from './crud/operations/find';
 import { GroupByOperationHandler } from './crud/operations/group-by';
 import { UpdateOperationHandler } from './crud/operations/update';
 import { InputValidator } from './crud/validator';
-import { createConfigError, createNotFoundError } from './errors';
+import { createConfigError, createNotFoundError, createNotSupportedError } from './errors';
 import { ZenStackDriver } from './executor/zenstack-driver';
 import { ZenStackQueryExecutor } from './executor/zenstack-query-executor';
 import * as BuiltinFunctions from './functions';
@@ -564,6 +564,11 @@ function createModelCrudHandler(
         },
 
         createManyAndReturn: (args: unknown) => {
+            if (client.$schema.provider.type === 'mysql') {
+                throw createNotSupportedError(
+                    '"createManyAndReturn" is not supported by "mysql" provider. Use "createMany" or multiple "create" calls instead.',
+                );
+            }
             return createPromise(
                 'createManyAndReturn',
                 'createManyAndReturn',
@@ -594,6 +599,11 @@ function createModelCrudHandler(
         },
 
         updateManyAndReturn: (args: unknown) => {
+            if (client.$schema.provider.type === 'mysql') {
+                throw createNotSupportedError(
+                    '"updateManyAndReturn" is not supported by "mysql" provider. Use "updateMany" or multiple "update" calls instead.',
+                );
+            }
             return createPromise(
                 'updateManyAndReturn',
                 'updateManyAndReturn',