Chore!: bump sqlglot to v26.17.1 (#4338)

Author: georgesittas

examples/sushi/models/marketing.sql

@@ -20,7 +20,5 @@ FROM
         customer_id: 'int',
         status: 'text',
         updated_at: 'timestamp',
-        valid_from: 'timestamp',
-        valid_to: 'timestamp',
     }
 )

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "requests",
     "rich[jupyter]",
     "ruamel.yaml",
-    "sqlglot[rs]~=26.16.3",
+    "sqlglot[rs]~=26.17.1",
     "tenacity",
     "time-machine",
     "json-stream"

tests/core/engine_adapter/integration/test_integration.py

@@ -1446,6 +1446,35 @@ def test_sushi(ctx: TestContext, tmp_path_factory: pytest.TempPathFactory):
                 f"CREATE VIEW {raw_test_schema}.demographics ON CLUSTER cluster1 AS SELECT 1 AS customer_id, '00000' AS zip;"
             )
 
+    # DuckDB parses TIMESTAMP into Type.TIMESTAMPNTZ which generates into TIMESTAMP_NTZ for
+    # Spark, but this type is not supported in Spark's DDL statements so we make it a TIMESTAMP
+    if ctx.dialect == "spark":
+        for model_key, model in context._models.items():
+            model_columns = model.columns_to_types
+
+            updated_model_columns = {}
+            for k, v in model_columns.items():
+                updated_model_columns[k] = v
+                if v.this == exp.DataType.Type.TIMESTAMPNTZ:
+                    v.set("this", exp.DataType.Type.TIMESTAMP)
+
+            update_fields = {
+                "columns_to_types": updated_model_columns,
+                "columns_to_types_": updated_model_columns,
+                "columns_to_types_or_raise": updated_model_columns,
+            }
+
+            # We get rid of the sushi.marketing post statement here because it asserts that
+            # updated_at is a 'timestamp', which is parsed using duckdb in assert_has_columns
+            # and the assertion fails because we now have TIMESTAMPs and not TIMESTAMPNTZs in
+            # the columns_to_types mapping
+            if '"marketing"' in model_key:
+                update_fields["post_statements_"] = []
+
+            context._models.update(
+                {model_key: context._models[model_key].copy(update=update_fields)}
+            )
+
     if ctx.dialect == "athena":
         for model_name in {"customer_revenue_lifetime"}:
             model_key = next(k for k in context._models if model_name in k)

tests/core/test_model.py

@@ -250,15 +250,15 @@ def test_model_union_query(sushi_context, assert_exp_eq):
         """SELECT
   CAST("marketing"."customer_id" AS INT) AS "customer_id",
   CAST("marketing"."status" AS TEXT) AS "status",
-  CAST("marketing"."updated_at" AS TIMESTAMP) AS "updated_at",
+  CAST("marketing"."updated_at" AS TIMESTAMPNTZ) AS "updated_at",
   CAST("marketing"."valid_from" AS TIMESTAMP) AS "valid_from",
   CAST("marketing"."valid_to" AS TIMESTAMP) AS "valid_to"
 FROM "memory"."sushi"."marketing" AS "marketing"
 UNION ALL
 SELECT
   CAST("marketing"."customer_id" AS INT) AS "customer_id",
   CAST("marketing"."status" AS TEXT) AS "status",
-  CAST("marketing"."updated_at" AS TIMESTAMP) AS "updated_at",
+  CAST("marketing"."updated_at" AS TIMESTAMPNTZ) AS "updated_at",
   CAST("marketing"."valid_from" AS TIMESTAMP) AS "valid_from",
   CAST("marketing"."valid_to" AS TIMESTAMP) AS "valid_to"
 FROM "memory"."sushi"."marketing" AS "marketing"