fix(introspection): preserve data validation attributes, enum ordering, and comments during db pull (#672)

* fix(introspection): preserve schema integrity during db pull

- Retain data validation attributes (e.g., @email) on fields after
  introspection (#670)
- Preserve original declaration order of enums instead of moving them
  to the end of the schema file (#669)
- Preserve triple-slash comments above enum declarations (#669)

Fixes #669, fixes #670

* fix: address PR comments

* fix(cli): improve db pull for relations and defaults

Prevents field name collisions during introspection by refining the naming strategy for self-referencing relations with multiple foreign keys.

Extends support for JSON and Bytes default values across MySQL, PostgreSQL, and SQLite providers to ensure consistent schema restoration.

Adds test cases for self-referencing models to verify the avoidance of duplicate fields.

* fix: address PR comments

Author: svetch

packages/cli/src/actions/db.ts

@@ -12,9 +12,9 @@ import {
     loadSchemaDocument,
     requireDataSourceUrl,
 } from './action-utils';
-import { syncEnums, syncRelation, syncTable, type Relation } from './pull';
+import { consolidateEnums, syncEnums, syncRelation, syncTable, type Relation } from './pull';
 import { providers as pullProviders } from './pull/provider';
-import { getDatasource, getDbName, getRelationFieldsKey, getRelationFkName } from './pull/utils';
+import { getDatasource, getDbName, getRelationFieldsKey, getRelationFkName, isDatabaseManagedAttribute } from './pull/utils';
 import type { DataSourceProviderType } from '@zenstackhq/schema';
 import { CliError } from '../cli-error';
 
@@ -173,6 +173,10 @@ async function runPull(options: PullOptions) {
             });
         }
 
+        // Consolidate per-column enums (e.g., MySQL's synthetic UserStatus/GroupStatus)
+        // back to shared enums from the original schema (e.g., Status)
+        consolidateEnums({ newModel, oldModel: model });
+
         console.log(colors.blue('Schema synced'));
 
         const baseDir = path.dirname(path.resolve(schemaFile));
@@ -457,12 +461,13 @@ async function runPull(options: PullOptions) {
                         }
                         return;
                     }
+
                     // Track deleted attributes (in original but not in new)
                     originalField.attributes
                         .filter(
                             (attr) =>
-                                !f.attributes.find((d) => d.decl.$refText === attr.decl.$refText) &&
-                                !['@map', '@@map', '@default', '@updatedAt'].includes(attr.decl.$refText),
+                                !f.attributes.find((d) => d.decl.$refText === attr.decl.$refText) && 
+                                isDatabaseManagedAttribute(attr.decl.$refText),
                         )
                         .forEach((attr) => {
                             const field = attr.$container;
@@ -478,7 +483,7 @@ async function runPull(options: PullOptions) {
                         .filter(
                             (attr) =>
                                 !originalField.attributes.find((d) => d.decl.$refText === attr.decl.$refText) &&
-                                !['@map', '@@map', '@default', '@updatedAt'].includes(attr.decl.$refText),
+                                isDatabaseManagedAttribute(attr.decl.$refText),
                         )
                         .forEach((attr) => {
                           // attach the new attribute to the original field
@@ -619,8 +624,8 @@ async function runPull(options: PullOptions) {
         }
 
         const generator = new ZModelCodeGenerator({
-            quote: options.quote,
-            indent: options.indent,
+            quote: options.quote ?? 'single',
+            indent: options.indent ?? 4,
         });
 
         if (options.output) {

packages/cli/src/actions/pull/index.ts

@@ -533,13 +533,20 @@ export function syncRelation({
     sourceModel.fields.splice(firstSourceFieldId, 0, sourceFieldFactory.node); // Insert the relation field before the first FK scalar field
 
     const oppositeFieldPrefix = /[0-9]/g.test(targetModel.name.charAt(0)) ? '_' : '';
-    const { name: oppositeFieldName } = resolveNameCasing(
+    let { name: oppositeFieldName } = resolveNameCasing(
         options.fieldCasing,
         similarRelations > 0
             ? `${oppositeFieldPrefix}${lowerCaseFirst(sourceModel.name)}_${firstColumn}`
             : `${lowerCaseFirst(resolveNameCasing(options.fieldCasing, sourceModel.name).name)}${relation.references.type === 'many'? 's' : ''}`,
     );
 
+    if (targetModel.fields.find((f) => f.name === oppositeFieldName)) {
+        ({ name: oppositeFieldName } = resolveNameCasing(
+            options.fieldCasing,
+            `${lowerCaseFirst(sourceModel.name)}_${firstColumn}To${relation.references.table}_${relation.references.columns[0]}`,
+        ));
+    }
+
     const targetFieldFactory = new DataFieldFactory()
         .setContainer(targetModel)
         .setName(oppositeFieldName)
@@ -556,3 +563,116 @@ export function syncRelation({
 
     targetModel.fields.push(targetFieldFactory.node);
 }
+
+/**
+ * Consolidates per-column enums back to shared enums when possible.
+ *
+ * MySQL doesn't have named enum types — each column gets a synthetic enum
+ * (e.g., `UserStatus`, `GroupStatus`). When the original schema used a shared
+ * enum (e.g., `Status`) across multiple fields, this function detects the
+ * mapping via field references and consolidates the synthetic enums back into
+ * the original shared enum so the merge phase can match them correctly.
+ */
+export function consolidateEnums({
+    newModel,
+    oldModel,
+}: {
+    newModel: Model;
+    oldModel: Model;
+}) {
+    const newEnums = newModel.declarations.filter((d) => isEnum(d)) as Enum[];
+    const newDataModels = newModel.declarations.filter((d) => d.$type === 'DataModel') as DataModel[];
+    const oldDataModels = oldModel.declarations.filter((d) => d.$type === 'DataModel') as DataModel[];
+
+    // For each new enum, find which old enum it corresponds to (via field references)
+    const enumMapping = new Map<Enum, Enum>(); // newEnum -> oldEnum
+
+    for (const newEnum of newEnums) {
+        for (const newDM of newDataModels) {
+            for (const field of newDM.fields) {
+                if (field.$type !== 'DataField' || field.type.reference?.ref !== newEnum) continue;
+
+                // Find matching model in old model by db name
+                const oldDM = oldDataModels.find((d) => getDbName(d) === getDbName(newDM));
+                if (!oldDM) continue;
+
+                // Find matching field in old model by db name
+                const oldField = oldDM.fields.find((f) => getDbName(f) === getDbName(field));
+                if (!oldField || oldField.$type !== 'DataField' || !oldField.type.reference?.ref) continue;
+
+                const oldEnum = oldField.type.reference.ref;
+                if (!isEnum(oldEnum)) continue;
+
+                enumMapping.set(newEnum, oldEnum as Enum);
+                break;
+            }
+            if (enumMapping.has(newEnum)) break;
+        }
+    }
+
+    // Group by old enum: oldEnum -> [newEnum1, newEnum2, ...]
+    const reverseMapping = new Map<Enum, Enum[]>();
+    for (const [newEnum, oldEnum] of enumMapping) {
+        if (!reverseMapping.has(oldEnum)) {
+            reverseMapping.set(oldEnum, []);
+        }
+        reverseMapping.get(oldEnum)!.push(newEnum);
+    }
+
+    // Consolidate: when new enums map to the same old enum with matching values
+    for (const [oldEnum, newEnumsGroup] of reverseMapping) {
+        const keepEnum = newEnumsGroup[0]!;
+
+        // Skip if already correct (single enum with matching name)
+        if (newEnumsGroup.length === 1 && keepEnum.name === oldEnum.name) continue;
+
+        // Check that all new enums have the same values as the old enum
+        const oldValues = new Set(oldEnum.fields.map((f) => getDbName(f)));
+        const allMatch = newEnumsGroup.every((ne) => {
+            const newValues = new Set(ne.fields.map((f) => getDbName(f)));
+            return oldValues.size === newValues.size && [...oldValues].every((v) => newValues.has(v));
+        });
+
+        if (!allMatch) continue;
+
+        // Rename the kept enum to match the old shared name
+        keepEnum.name = oldEnum.name;
+
+        // Replace keepEnum's attributes with those from the old enum so that
+        // any synthetic @@map added by syncEnums is removed and getDbName(keepEnum)
+        // reflects the consolidated name rather than the stale per-column name.
+        // Shallow-copy and re-parent so AST $container pointers reference keepEnum.
+        keepEnum.attributes = oldEnum.attributes.map((attr) => {
+            const copy = { ...attr, $container: keepEnum };
+            return copy;
+        });
+
+        // Remove duplicate enums from newModel
+        for (let i = 1; i < newEnumsGroup.length; i++) {
+            const idx = newModel.declarations.indexOf(newEnumsGroup[i]!);
+            if (idx >= 0) {
+                newModel.declarations.splice(idx, 1);
+            }
+        }
+
+        // Update all field references in newModel to point to the kept enum
+        for (const newDM of newDataModels) {
+            for (const field of newDM.fields) {
+                if (field.$type !== 'DataField') continue;
+                const ref = field.type.reference?.ref;
+                if (ref && newEnumsGroup.includes(ref as Enum)) {
+                    (field.type as any).reference = {
+                        ref: keepEnum,
+                        $refText: keepEnum.name,
+                    };
+                }
+            }
+        }
+
+        console.log(
+            colors.gray(
+                `Consolidated enum${newEnumsGroup.length > 1 ? 's' : ''} ${newEnumsGroup.map((e) => e.name).join(', ')} → ${oldEnum.name}`,
+            ),
+        );
+    }
+}

packages/cli/src/actions/pull/provider/mysql.ts

@@ -266,6 +266,10 @@ export const mysql: IntrospectionProvider = {
                     return (ab) => ab.InvocationExpr.setFunction(getFunctionRef('uuid', services));
                 }
                 return (ab) => ab.StringLiteral.setValue(val);
+            case 'Json':
+                return (ab) => ab.StringLiteral.setValue(val);
+            case 'Bytes':
+                return (ab) => ab.StringLiteral.setValue(val);
         }
 
         // Handle function calls (e.g., uuid(), now())

packages/cli/src/actions/pull/provider/postgresql.ts

@@ -284,6 +284,16 @@ export const postgresql: IntrospectionProvider = {
                     return (ab) => ab.StringLiteral.setValue(val.slice(1, -1).replace(/''/g, "'"));
                 }
                 return (ab) => ab.StringLiteral.setValue(val);
+            case 'Json':
+                if (val.includes('::')) {
+                    return typeCastingConvert({defaultValue,enums,val,services});
+                }
+                return (ab) => ab.StringLiteral.setValue(val);
+            case 'Bytes':
+                if (val.includes('::')) {
+                    return typeCastingConvert({defaultValue,enums,val,services});
+                }
+                return (ab) => ab.StringLiteral.setValue(val);
         }
 
         if (val.includes('(') && val.includes(')')) {

packages/cli/src/actions/pull/provider/sqlite.ts

@@ -394,6 +394,10 @@ export const sqlite: IntrospectionProvider = {
                     return (ab) => ab.StringLiteral.setValue(strippedName);
                 }
                 return (ab) => ab.StringLiteral.setValue(val);
+            case 'Json':
+                return (ab) => ab.StringLiteral.setValue(val);
+            case 'Bytes':
+                return (ab) => ab.StringLiteral.setValue(val);
         }
 
         console.warn(`Unsupported default value type: "${defaultValue}" for field type "${fieldType}". Skipping default value.`);

packages/cli/src/actions/pull/utils.ts

@@ -28,6 +28,10 @@ export function getAttribute(model: Model, attrName: string) {
         | undefined;
 }
 
+export function isDatabaseManagedAttribute(name: string) {
+    return ['@relation', '@id', '@unique'].includes(name) || name.startsWith('@db.');
+}
+
 export function getDatasource(model: Model) {
     const datasource = model.declarations.find((d) => d.$type === 'DataSource');
     if (!datasource) {

packages/cli/src/index.ts

@@ -170,7 +170,7 @@ function createProgram() {
         .addOption(
             new Option('--quote <double|single>', 'set the quote style of generated schema files').default('single'),
         )
-        .addOption(new Option('--indent <number>', 'set the indentation of the generated schema files').default(4).argParser(parseInt))
+        .addOption(new Option('--indent <number>', 'set the indentation of the generated schema files').default(4))
         .action((options) => dbAction('pull', options));
 
     dbCommand