feat(cli): implement watch mode for generate (#554)

* feat(cli): implement watch mode for generate

* chore(root): update pnpm-lock.yaml

* chore(cli): track all model declaration and removed paths, logs in past tense

* fix(cli): typo, unused double array from

* fix(orm): preserve zod validation errors when validating custom json types

* update

* chore(cli): move import, fix parallel generation on watch

* feat(common-helpers): implement single-debounce

* chore(cli): use single-debounce for debouncing

* feat(common-helpers): implement single-debounce

* fix(common-helpers): re run single-debounce

---------

Co-authored-by: Yiming Cao <yiming@whimslab.io>
Co-authored-by: ymc9 <104139426+ymc9@users.noreply.github.com>

Author: DoctorFTB

packages/cli/package.json

@@ -39,6 +39,7 @@
         "@zenstackhq/common-helpers": "workspace:*",
         "@zenstackhq/language": "workspace:*",
         "@zenstackhq/sdk": "workspace:*",
+        "chokidar": "^5.0.0",
         "colors": "1.4.0",
         "commander": "^8.3.0",
         "execa": "^9.6.0",

packages/cli/src/actions/generate.ts

@@ -1,12 +1,14 @@
-import { invariant } from '@zenstackhq/common-helpers';
-import { isPlugin, LiteralExpr, Plugin, type Model } from '@zenstackhq/language/ast';
+import { invariant, singleDebounce } from '@zenstackhq/common-helpers';
+import { ZModelLanguageMetaData } from '@zenstackhq/language';
+import { type AbstractDeclaration, isPlugin, LiteralExpr, Plugin, type Model } from '@zenstackhq/language/ast';
 import { getLiteral, getLiteralArray } from '@zenstackhq/language/utils';
 import { type CliPlugin } from '@zenstackhq/sdk';
 import colors from 'colors';
 import { createJiti } from 'jiti';
 import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
+import { watch } from 'chokidar';
 import ora, { type Ora } from 'ora';
 import { CliError } from '../cli-error';
 import * as corePlugins from '../plugins';
@@ -16,6 +18,7 @@ type Options = {
     schema?: string;
     output?: string;
     silent: boolean;
+    watch: boolean;
     lite: boolean;
     liteOnly: boolean;
 };
@@ -24,6 +27,96 @@ type Options = {
  * CLI action for generating code from schema
  */
 export async function run(options: Options) {
+    const model = await pureGenerate(options, false);
+
+    if (options.watch) {
+        const logsEnabled = !options.silent;
+
+        if (logsEnabled) {
+            console.log(colors.green(`\nEnabled watch mode!`));
+        }
+
+        const schemaExtensions = ZModelLanguageMetaData.fileExtensions;
+
+        // Get real models file path (cuz its merged into single document -> we need use cst nodes)
+        const getRootModelWatchPaths = (model: Model) => new Set<string>(
+            (
+                model.declarations.filter(
+                    (v) =>
+                        v.$cstNode?.parent?.element.$type === 'Model' &&
+                        !!v.$cstNode.parent.element.$document?.uri?.fsPath,
+                ) as AbstractDeclaration[]
+            ).map((v) => v.$cstNode!.parent!.element.$document!.uri!.fsPath),
+        );
+
+        const watchedPaths = getRootModelWatchPaths(model);
+
+        if (logsEnabled) {
+            const logPaths = [...watchedPaths].map((at) => `- ${at}`).join('\n');
+            console.log(`Watched file paths:\n${logPaths}`);
+        }
+
+        const watcher = watch([...watchedPaths], {
+            alwaysStat: false,
+            ignoreInitial: true,
+            ignorePermissionErrors: true,
+            ignored: (at) => !schemaExtensions.some((ext) => at.endsWith(ext)),
+        });
+
+        // prevent save multiple files and run multiple times
+        const reGenerateSchema = singleDebounce(async () => {
+            if (logsEnabled) {
+                console.log('Got changes, run generation!');
+            }
+
+            try {
+                const newModel = await pureGenerate(options, true);
+                const allModelsPaths = getRootModelWatchPaths(newModel);
+                const newModelPaths = [...allModelsPaths].filter((at) => !watchedPaths.has(at));
+                const removeModelPaths = [...watchedPaths].filter((at) => !allModelsPaths.has(at));
+
+                if (newModelPaths.length) {
+                    if (logsEnabled) {
+                        const logPaths = newModelPaths.map((at) => `- ${at}`).join('\n');
+                        console.log(`Added file(s) to watch:\n${logPaths}`);
+                    }
+
+                    newModelPaths.forEach((at) => watchedPaths.add(at));
+                    watcher.add(newModelPaths);
+                }
+
+                if (removeModelPaths.length) {
+                    if (logsEnabled) {
+                        const logPaths = removeModelPaths.map((at) => `- ${at}`).join('\n');
+                        console.log(`Removed file(s) from watch:\n${logPaths}`);
+                    }
+
+                    removeModelPaths.forEach((at) => watchedPaths.delete(at));
+                    watcher.unwatch(removeModelPaths);
+                }
+            } catch (e) {
+                console.error(e);
+            }
+        }, 500, true);
+
+        watcher.on('unlink', (pathAt) => {
+            if (logsEnabled) {
+                console.log(`Removed file from watch: ${pathAt}`);
+            }
+
+            watchedPaths.delete(pathAt);
+            watcher.unwatch(pathAt);
+
+            reGenerateSchema();
+        });
+
+        watcher.on('change', () => {
+            reGenerateSchema();
+        });
+    }
+}
+
+async function pureGenerate(options: Options, fromWatch: boolean) {
     const start = Date.now();
 
     const schemaFile = getSchemaFile(options.schema);
@@ -35,7 +128,9 @@ export async function run(options: Options) {
 
     if (!options.silent) {
         console.log(colors.green(`Generation completed successfully in ${Date.now() - start}ms.\n`));
-        console.log(`You can now create a ZenStack client with it.
+
+        if (!fromWatch) {
+            console.log(`You can now create a ZenStack client with it.
 
 \`\`\`ts
 import { ZenStackClient } from '@zenstackhq/orm';
@@ -47,7 +142,10 @@ const client = new ZenStackClient(schema, {
 \`\`\`
 
 Check documentation: https://zenstack.dev/docs/`);
+        }
     }
+
+    return model;
 }
 
 function getOutputPath(options: Options, schemaFile: string) {

packages/cli/src/index.ts

@@ -68,6 +68,7 @@ function createProgram() {
         .addOption(schemaOption)
         .addOption(noVersionCheckOption)
         .addOption(new Option('-o, --output <path>', 'default output directory for code generation'))
+        .addOption(new Option('-w, --watch', 'enable watch mode').default(false))
         .addOption(new Option('--lite', 'also generate a lite version of schema without attributes').default(false))
         .addOption(new Option('--lite-only', 'only generate lite version of schema without attributes').default(false))
         .addOption(new Option('--silent', 'suppress all output except errors').default(false))
@@ -220,6 +221,11 @@ async function main() {
         }
     }
 
+    if (program.args.includes('generate') && (program.args.includes('-w') || program.args.includes('--watch'))) {
+        // A "hack" way to prevent the process from terminating because we don't want to stop it.
+        return;
+    }
+
     if (telemetry.isTracking) {
         // give telemetry a chance to send events before exit
         setTimeout(() => {

packages/common-helpers/src/index.ts

@@ -4,6 +4,7 @@ export * from './is-plain-object';
 export * from './lower-case-first';
 export * from './param-case';
 export * from './safe-json-stringify';
+export * from './single-debounce';
 export * from './sleep';
 export * from './tiny-invariant';
 export * from './upper-case-first';

packages/common-helpers/src/single-debounce.ts

@@ -0,0 +1,34 @@
+export function singleDebounce(cb: () => void | PromiseLike<void>, debounceMc: number, reRunOnInProgressCall: boolean = false) {
+    let timeout: ReturnType<typeof setTimeout> | undefined;
+    let inProgress = false;
+    let pendingInProgress = false;
+
+    const run = async () => {
+        if (inProgress) {
+            if (reRunOnInProgressCall) {
+                pendingInProgress = true;
+            }
+
+            return;
+        }
+
+        inProgress = true;
+        pendingInProgress = false;
+
+        try {
+            await cb();
+        } finally {
+            inProgress = false;
+
+            if (pendingInProgress) {
+                await run();
+            }
+        }
+    };
+
+    return () => {
+        clearTimeout(timeout);
+
+        timeout = setTimeout(run, debounceMc);
+    }
+}

packages/orm/src/client/crud/validator/index.ts

@@ -402,9 +402,13 @@ export class InputValidator<Schema extends SchemaDef> {
         // zod doesn't preserve object field order after parsing, here we use a
         // validation-only custom schema and use the original data if parsing
         // is successful
-        const finalSchema = z.custom((v) => {
-            return schema.safeParse(v).success;
+        const finalSchema = z.any().superRefine((value, ctx) => {
+            const parseResult = schema.safeParse(value);
+            if (!parseResult.success) {
+                parseResult.error.issues.forEach((issue) => ctx.addIssue(issue as any));
+            }
         });
+
         this.setSchemaCache(key!, finalSchema);
         return finalSchema;
     }
@@ -515,7 +519,7 @@ export class InputValidator<Schema extends SchemaDef> {
         }
 
         // expression builder
-        fields['$expr'] = z.custom((v) => typeof v === 'function').optional();
+        fields['$expr'] = z.custom((v) => typeof v === 'function', { error: '"$expr" must be a function' }).optional();
 
         // logical operators
         fields['AND'] = this.orArray(

pnpm-lock.yaml

@@ -121,7 +121,7 @@ model User {
                     },
                 },
             }),
-        ).rejects.toThrow(/invalid/i);
+        ).rejects.toThrow('data.identity.providers[0].id');
     });
 
     it('works with find', async () => {

tests/e2e/orm/client-api/typed-json-fields.test.ts

@@ -0,0 +1,19 @@
+import { createTestClient } from '@zenstackhq/testtools';
+import { describe, expect, it } from 'vitest';
+
+describe('Regression for issue #558', () => {
+    it('verifies issue 558', async () => {
+        const db = await createTestClient(`
+type Foo {
+    x Int
+}
+
+model Model {
+    id String @id @default(cuid())
+    foo Foo @json
+}
+        `);
+
+        await expect(db.model.create({ data: { foo: { x: 'hello' } } })).rejects.toThrow('data.foo.x');
+    });
+});