Feat: Render should not quote identifiers by default (#726)

* Do not add quotes to non-quoted identifiers when rendering

* Fix migrate magic

* First stab at migrations documentation

* Remove quoting in expected results

* fix quotes for schema diff

---------

Co-authored-by: eakmanrq <6326532+eakmanrq@users.noreply.github.com>

Author: crericha

docs/guides/migrations.md

@@ -0,0 +1,31 @@
+# Migrations guide
+
+New versions of SQLMesh may be incompatible with the project's stored metadata format. Migrations provide a way to upgrade the project metadata format to operate with the new SQLMesh version.
+
+## Detecting incompatibility
+When issuing a SQLMesh command, SQLMesh will automatically check for incompatibilities between the installed version of SQLMesh and the project's metadata format, prompting what action is required. SQLMesh commands will not execute until the action is complete.
+
+### Installed version is newer than metadata format
+In this scenario, the project's metadata format needs to be migrated.
+
+```bash
+> sqlmesh plan my_dev
+Error: SQLMesh (local) is using version '2' which is ahead of '1' (remote). Please run a migration.
+```
+
+### Installed version is older than metadata format
+Here, the installed version of SQLMesh needs to be upgraded.
+
+```bash
+> sqlmesh plan my_dev
+SQLMeshError: SQLMesh (local) is using version '1' which is behind '2' (remote). Please upgrade SQLMesh.
+```
+
+## How to migrate
+The project metadata can be migrated to the latest metadata format using SQLMesh's migrate command.
+
+```bash
+> sqlmesh migrate
+```
+
+Migrating project metadata will affect all users of the project. Please consult your SQLMesh administrator before running the migrate command.

docs/reference/cli.md

@@ -18,6 +18,7 @@ Commands:
   fetchdf   Runs a sql query and displays the results.
   format    Format all models in a given directory.
   init      Create a new SQLMesh repository.
+  migrate   Migrate SQLMesh to the current running version.
   plan      Plan a migration of the current context's models with the...
   render    Renders a model's query, optionally expanding referenced models.
   run       Evaluates the DAG of models using the built-in scheduler.
@@ -169,3 +170,12 @@ Options:
   --file TEXT  The file to which the dag image should be written.
   --help       Show this message and exit.
 ```
+
+## migrate
+```
+Usage: sqlmesh migrate
+
+  Migrates SQLMesh to the current running version.
+  
+  Please contact your SQLMesh administrator before doing this.
+```

mkdocs.yml

@@ -12,6 +12,7 @@ nav:
     - guides/scheduling.md
     - guides/connections.md
     - guides/multi_repo.md
+    - guides/migrations.md
   - Concepts:
     - concepts/overview.md
     - Models:

sqlmesh/cli/main.py

@@ -342,5 +342,13 @@ def ide(
     uvicorn.run("web.server.main:app", host=host, port=port, log_level="info")
 
 
+@cli.command("migrate")
+@click.pass_context
+@error_handler
+def migrate(ctx: click.Context) -> None:
+    """Migrate SQLMesh to the current running version."""
+    ctx.obj.migrate()
+
+
 if __name__ == "__main__":
     cli()

sqlmesh/core/engine_adapter/base.py

@@ -777,7 +777,6 @@ def _to_sql(self, e: exp.Expression, **kwargs: t.Any) -> str:
             "dialect": self.dialect,
             "pretty": False,
             "comments": False,
-            "identify": True,
             **self.DEFAULT_SQL_GEN_KWARGS,
             **self.sql_gen_kwargs,
             **kwargs,

sqlmesh/core/engine_adapter/databricks.py

@@ -9,5 +9,5 @@ class DatabricksSparkSessionEngineAdapter(SparkEngineAdapter):
     SCHEMA_DIFFER = SchemaDiffer(
         support_positional_add=True,
         support_nested_operations=True,
-        array_suffix=".element",
+        array_element_selector="element",
     )

sqlmesh/core/engine_adapter/databricks_api.py

@@ -17,7 +17,7 @@ class DatabricksSQLEngineAdapter(BaseSparkEngineAdapter):
     SCHEMA_DIFFER = SchemaDiffer(
         support_positional_add=True,
         support_nested_operations=True,
-        array_suffix=".element",
+        array_element_selector="element",
     )
 
     def _fetch_native_df(self, query: t.Union[exp.Expression, str]) -> DF:

sqlmesh/core/engine_adapter/postgres.py

@@ -42,7 +42,7 @@ def replace_query(
         if not self.table_exists(table_name):
             return self.ctas(table_name, query_or_df, columns_to_types, exists=False)
         with self.transaction(TransactionType.DDL):
-            sql = f"TRUNCATE {exp.to_table(table_name).sql(identify=True, dialect=self.dialect)}"
+            sql = f"TRUNCATE {exp.to_table(table_name).sql(dialect=self.dialect)}"
             self.execute(sql)
             return self.insert_append(table_name, query_or_df, columns_to_types)
 

sqlmesh/core/schema_diff.py

@@ -32,43 +32,61 @@ class TableAlterColumn(PydanticModel):
     is_struct: bool
     is_array_of_struct: bool
     is_array_of_primitive: bool
+    quoted: bool = False
 
     @classmethod
-    def primitive(self, name: str) -> TableAlterColumn:
+    def primitive(self, name: str, quoted: bool = False) -> TableAlterColumn:
         return self(
-            name=name, is_struct=False, is_array_of_struct=False, is_array_of_primitive=False
+            name=name,
+            is_struct=False,
+            is_array_of_struct=False,
+            is_array_of_primitive=False,
+            quoted=quoted,
         )
 
     @classmethod
-    def struct(self, name: str) -> TableAlterColumn:
+    def struct(self, name: str, quoted: bool = False) -> TableAlterColumn:
         return self(
-            name=name, is_struct=True, is_array_of_struct=False, is_array_of_primitive=False
+            name=name,
+            is_struct=True,
+            is_array_of_struct=False,
+            is_array_of_primitive=False,
+            quoted=quoted,
         )
 
     @classmethod
-    def array_of_struct(self, name: str) -> TableAlterColumn:
+    def array_of_struct(self, name: str, quoted: bool = False) -> TableAlterColumn:
         return self(
-            name=name, is_struct=False, is_array_of_struct=True, is_array_of_primitive=False
+            name=name,
+            is_struct=False,
+            is_array_of_struct=True,
+            is_array_of_primitive=False,
+            quoted=quoted,
         )
 
     @classmethod
-    def array_of_primitive(self, name: str) -> TableAlterColumn:
+    def array_of_primitive(self, name: str, quoted: bool = False) -> TableAlterColumn:
         return self(
-            name=name, is_struct=False, is_array_of_struct=False, is_array_of_primitive=True
+            name=name,
+            is_struct=False,
+            is_array_of_struct=False,
+            is_array_of_primitive=True,
+            quoted=quoted,
         )
 
     @classmethod
     def from_struct_kwarg(self, struct: exp.StructKwarg) -> TableAlterColumn:
         name = struct.alias_or_name
+        quoted = struct.this.quoted
         if struct.expression.is_type(exp.DataType.Type.STRUCT):
-            return self.struct(name)
+            return self.struct(name, quoted=quoted)
         elif struct.expression.is_type(exp.DataType.Type.ARRAY):
             if struct.expression.expressions[0].is_type(exp.DataType.Type.STRUCT):
-                return self.array_of_struct(name)
+                return self.array_of_struct(name, quoted=quoted)
             else:
-                return self.array_of_primitive(name)
+                return self.array_of_primitive(name, quoted=quoted)
         else:
-            return self.primitive(name)
+            return self.primitive(name, quoted=quoted)
 
     @property
     def is_array(self) -> bool:
@@ -82,23 +100,29 @@ def is_primitive(self) -> bool:
     def is_nested(self) -> bool:
         return not self.is_primitive
 
+    @property
+    def identifier(self) -> exp.Identifier:
+        return exp.to_identifier(self.name, quoted=self.quoted)
+
 
 class TableAlterColumnPosition(PydanticModel):
     is_first: bool
     is_last: bool
-    after: t.Optional[str] = None
+    after: t.Optional[exp.Identifier] = None
 
     @classmethod
     def first(self) -> TableAlterColumnPosition:
         return self(is_first=True, is_last=False, after=None)
 
     @classmethod
-    def last(self, after: t.Optional[str] = None) -> TableAlterColumnPosition:
-        return self(is_first=False, is_last=True, after=after)
+    def last(
+        self, after: t.Optional[t.Union[str, exp.Identifier]] = None
+    ) -> TableAlterColumnPosition:
+        return self(is_first=False, is_last=True, after=exp.to_identifier(after) if after else None)
 
     @classmethod
-    def middle(self, after: str) -> TableAlterColumnPosition:
-        return self(is_first=False, is_last=False, after=after)
+    def middle(self, after: t.Union[str, exp.Identifier]) -> TableAlterColumnPosition:
+        return self(is_first=False, is_last=False, after=exp.to_identifier(after))
 
     @classmethod
     def create(
@@ -117,7 +141,7 @@ def create(
 
     @property
     def column_position_node(self) -> t.Optional[exp.ColumnPosition]:
-        column = exp.column(self.after) if self.after and not self.is_last else None
+        column = self.after if not self.is_last else None
         position = None
         if self.is_first:
             position = "FIRST"
@@ -195,45 +219,49 @@ def is_drop(self) -> bool:
     def is_alter_type(self) -> bool:
         return self.op.is_alter_type
 
-    def full_column_path(self, array_suffix: str) -> str:
+    def column_identifiers(self, array_element_selector: str) -> t.List[exp.Identifier]:
         results = []
         for column in self.columns:
-            if column.is_array_of_struct and len(self.columns) > 1:
-                results.append(column.name + array_suffix)
-            else:
-                results.append(column.name)
-        return ".".join(results)
-
-    def column(self, array_suffix: str) -> exp.Column:
-        return exp.column(self.full_column_path(array_suffix))
-
-    def column_def(self, array_suffix: str) -> exp.ColumnDef:
+            results.append(column.identifier)
+            if column.is_array_of_struct and len(self.columns) > 1 and array_element_selector:
+                results.append(exp.to_identifier(array_element_selector))
+        return results
+
+    def column(self, array_element_selector: str) -> t.Union[exp.Dot, exp.Identifier]:
+        columns = self.column_identifiers(array_element_selector)
+        if len(columns) == 1:
+            return columns[0]
+        return exp.Dot.build(columns)
+
+    def column_def(self, array_element_selector: str) -> exp.ColumnDef:
         return exp.ColumnDef(
-            this=exp.to_identifier(self.full_column_path(array_suffix)),
+            this=self.column(array_element_selector),
             kind=self.column_type,
         )
 
-    def expression(self, table_name: t.Union[str, exp.Table], array_suffix: str) -> exp.AlterTable:
+    def expression(
+        self, table_name: t.Union[str, exp.Table], array_element_selector: str
+    ) -> exp.AlterTable:
         if self.is_alter_type:
             return exp.AlterTable(
                 this=exp.to_table(table_name),
                 actions=[
                     exp.AlterColumn(
-                        this=self.column(array_suffix),
+                        this=self.column(array_element_selector),
                         dtype=self.column_type,
                     )
                 ],
             )
         elif self.is_add:
             alter_table = exp.AlterTable(this=exp.to_table(table_name))
-            column = self.column_def(array_suffix)
+            column = self.column_def(array_element_selector)
             alter_table.set("actions", [column])
             if self.add_position:
                 column.set("position", self.add_position.column_position_node)
             return alter_table
         elif self.is_drop:
             alter_table = exp.AlterTable(this=exp.to_table(table_name))
-            drop_column = exp.Drop(this=self.column(array_suffix), kind="COLUMN")
+            drop_column = exp.Drop(this=self.column(array_element_selector), kind="COLUMN")
             alter_table.set("actions", [drop_column])
             return alter_table
         else:
@@ -260,7 +288,7 @@ class SchemaDiffer(PydanticModel):
 
     support_positional_add: bool = False
     support_nested_operations: bool = False
-    array_suffix: str = ""
+    array_element_selector: str = ""
     compatible_types: t.Dict[exp.DataType, t.Set[exp.DataType]] = {}
 
     @classmethod
@@ -287,20 +315,20 @@ def _get_matching_kwarg(
         current_pos: int,
     ) -> t.Tuple[t.Optional[int], t.Optional[exp.StructKwarg]]:
         current_name = (
-            current_kwarg
+            exp.to_identifier(current_kwarg)
             if isinstance(current_kwarg, str)
             else _get_name_and_type(current_kwarg)[0]
         )
         # First check if we have the same column in the same position to get O(1) complexity
         new_kwarg = seq_get(new_struct.expressions, current_pos)
         if new_kwarg:
             new_name, new_type = _get_name_and_type(new_kwarg)
-            if current_name == new_name:
+            if current_name.this == new_name.this:
                 return current_pos, new_kwarg
         # If not, check if we have the same column in all positions with O(n) complexity
         for i, new_kwarg in enumerate(new_struct.expressions):
             new_name, new_type = _get_name_and_type(new_kwarg)
-            if current_name == new_name:
+            if current_name.this == new_name.this:
                 return i, new_kwarg
         return None, None
 
@@ -499,7 +527,8 @@ def compare_structs(
             The list of table alter operations.
         """
         return [
-            op.expression(table_name, self.array_suffix) for op in self._from_structs(current, new)
+            op.expression(table_name, self.array_element_selector)
+            for op in self._from_structs(current, new)
         ]
 
     def compare_columns(
@@ -523,5 +552,5 @@ def compare_columns(
         )
 
 
-def _get_name_and_type(struct: exp.StructKwarg) -> t.Tuple[str, exp.DataType]:
-    return struct.alias_or_name, struct.expression
+def _get_name_and_type(struct: exp.StructKwarg) -> t.Tuple[exp.Identifier, exp.DataType]:
+    return struct.this, struct.expression

sqlmesh/core/snapshot/definition.py

@@ -733,10 +733,7 @@ def serialize_hooks(hooks: t.List[HookCall]) -> t.Iterable[str]:
                 name, args = hook
                 serialized.append(
                     f"{name}:"
-                    + ",".join(
-                        f"{k}={v.sql(identify=True, comments=False)}"
-                        for k, v in sorted(args.items())
-                    )
+                    + ",".join(f"{k}={v.sql(comments=False)}" for k, v in sorted(args.items()))
                 )
         return serialized
 
@@ -747,14 +744,14 @@ def serialize_hooks(hooks: t.List[HookCall]) -> t.Iterable[str]:
         model.storage_format,
         physical_schema,
         *(model.partitioned_by or []),
-        *(expression.sql(identify=True, comments=False) for expression in model.expressions or []),
+        *(expression.sql(comments=False) for expression in model.expressions or []),
         *serialize_hooks(model.pre),
         *serialize_hooks(model.post),
         model.stamp,
     ]
 
     if isinstance(model, SqlModel):
-        data.append(model.query.sql(identify=True, comments=False))
+        data.append(model.query.sql(comments=False))
 
         for macro_name, macro in sorted(model.jinja_macros.root_macros.items(), key=lambda x: x[0]):
             data.append(macro_name)
@@ -800,13 +797,13 @@ def _model_metadata_hash(model: Model, audits: t.Dict[str, Audit]) -> str:
         if audit_name in BUILT_IN_AUDITS:
             for arg_name, arg_value in audit_args.items():
                 metadata.append(arg_name)
-                metadata.append(arg_value.sql(identify=True, comments=True))
+                metadata.append(arg_value.sql(comments=True))
         elif audit_name in audits:
             audit = audits[audit_name]
             metadata.extend(
                 [
                     audit.render_query(model, **t.cast(t.Dict[str, t.Any], audit_args)).sql(
-                        identify=True, comments=True
+                        comments=True
                     ),
                     audit.dialect,
                     str(audit.skip),