update test for doris

Author: xinge-ji

.circleci/wait-for-db.sh

@@ -36,20 +36,26 @@ clickhouse_ready() {
 
 doris_ready() {
     probe_port 9030
-    
-    # Check that we have 3 alive backends
+
     echo "Checking for 3 alive Doris backends..."
-    
+
     while true; do
         echo "Checking Doris backends..."
-        # Use docker compose exec to run mysql command inside the fe-01 container
-        # Use timeout to prevent hanging, and handle connection errors gracefully
-        ALIVE_BACKENDS=$(timeout 10 docker compose -f tests/core/engine_adapter/integration/docker/compose.doris.yaml exec -T fe-01 mysql -uroot -e "show backends \G" 2>/dev/null | grep -c "Alive: true" || echo "0")
+        ALIVE_BACKENDS=$(timeout 10 docker compose exec -T doris-fe-01 mysql -uroot -e "show backends \G" 2>/dev/null | grep -c "Alive: true")
+        
+        # fallback value if failed to get number
+        if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then
+            echo "WARN: Unable to parse number of alive backends, got: '$ALIVE_BACKENDS'"
+            ALIVE_BACKENDS=0
+        fi
+
         echo "Found $ALIVE_BACKENDS alive backends"
+
         if [ "$ALIVE_BACKENDS" -ge 3 ]; then
             echo "Doris has 3 or more alive backends"
             break
         fi
+
         echo "Waiting for more backends to become alive..."
         sleep 5
     done

pyproject.toml

@@ -78,6 +78,7 @@ dev = [
     "pydantic",
     "PyAthena[Pandas]",
     "PyGithub>=2.6.0",
+    "pymysql",
     "pyodbc>=5.0.0",
     "pyperf",
     "pyspark~=3.5.0",

sqlmesh/cli/project_init.py

@@ -7,7 +7,6 @@
 from sqlmesh.integrations.dlt import generate_dlt_models_and_settings
 from sqlmesh.utils.date import yesterday_ds
 from sqlmesh.utils.errors import SQLMeshError
-from datetime import datetime
 
 from sqlmesh.core.config.connection import (
     CONNECTION_CONFIG_TO_TYPE,
@@ -161,7 +160,7 @@ class ExampleObjects:
     python_macros: t.Dict[str, str]
 
 
-def _gen_example_objects(schema_name: str, start_year: int) -> ExampleObjects:
+def _gen_example_objects(schema_name: str, dialect: str) -> ExampleObjects:
     sql_models: t.Dict[str, str] = {}
     python_models: t.Dict[str, str] = {}
     seeds: t.Dict[str, str] = {}
@@ -193,9 +192,10 @@ def _gen_example_objects(schema_name: str, start_year: int) -> ExampleObjects:
     sql_models[incremental_model_name] = f"""MODEL (
   name {incremental_model_name},
   kind INCREMENTAL_BY_TIME_RANGE (
-    time_column event_date
+    time_column event_date,
+    {"partition_by_time_column false" if dialect == "doris" else ""}
   ),
-  start '{start_year}-01-01',
+  start '2020-01-01',
   cron '@daily',
   grain (id, event_date)
 );
@@ -224,14 +224,14 @@ def _gen_example_objects(schema_name: str, start_year: int) -> ExampleObjects:
 );
   """
 
-    seeds["seed_data"] = f"""id,item_id,event_date
-1,2,{start_year}-01-01
-2,1,{start_year}-01-01
-3,3,{start_year}-01-03
-4,1,{start_year}-01-04
-5,1,{start_year}-01-05
-6,1,{start_year}-01-06
-7,1,{start_year}-01-07
+    seeds["seed_data"] = """id,item_id,event_date
+1,2,2020-01-01
+2,1,2020-01-01
+3,3,2020-01-03
+4,1,2020-01-04
+5,1,2020-01-05
+6,1,2020-01-06
+7,1,2020-01-07
 """
 
     audits["assert_positive_order_ids"] = """AUDIT (
@@ -319,7 +319,7 @@ def init_example_project(
     settings = None
     start = None
     if engine_type and template == ProjectTemplate.DLT:
-        project_dialect = dialect or DIALECT_TO_TYPE.get(engine_type)
+        project_dialect = dialect if dialect else DIALECT_TO_TYPE.get(engine_type)
         if pipeline and project_dialect:
             dlt_models, settings, start = generate_dlt_models_and_settings(
                 pipeline_name=pipeline, dialect=project_dialect, dlt_path=dlt_path
@@ -329,9 +329,6 @@ def init_example_project(
                 "Please provide a DLT pipeline with the `--dlt-pipeline` flag to generate a SQLMesh project from DLT."
             )
 
-        if engine_type == "doris":
-            start = datetime(datetime.now().year, 1, 1).strftime("%Y-%m-%d")
-
     _create_config(config_path, engine_type, dialect, settings, start, template, cli_mode)
     if template == ProjectTemplate.DBT:
         return config_path
@@ -344,7 +341,10 @@ def init_example_project(
         )
         return config_path
 
-    example_objects = _gen_example_objects(schema_name=schema_name, start_year=datetime.now().year)
+    example_objects = _gen_example_objects(
+        schema_name=schema_name,
+        dialect=dialect if dialect else DIALECT_TO_TYPE.get(engine_type, "duckdb"),
+    )
 
     if template != ProjectTemplate.EMPTY:
         _create_object_files(models_path, example_objects.sql_models, "sql")

sqlmesh/core/model/meta.py

@@ -48,6 +48,10 @@
 
 FunctionCall = t.Tuple[str, t.Dict[str, exp.Expression]]
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 class ModelMeta(_Node):
     """Metadata for models which can be defined in SQL."""
@@ -475,12 +479,9 @@ def physical_properties(self) -> t.Dict[str, exp.Expression]:
             and self.unique_key
             and self.dialect == "doris"
         ):
-            # Convert unique_key expressions to a format suitable for table_properties
             if len(self.unique_key) == 1:
-                # Single column key
                 properties["unique_key"] = self.unique_key[0]
             else:
-                # Multiple column key - create a tuple expression
                 properties["unique_key"] = exp.Tuple(expressions=self.unique_key)
 
         return properties

tests/core/engine_adapter/integration/test_integration.py

@@ -44,6 +44,10 @@
 DATA_TYPE = exp.DataType.Type
 VARCHAR_100 = exp.DataType.build("varchar(100)")
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 class PlanResults(PydanticModel):
     plan: Plan
@@ -1920,6 +1924,7 @@ def _mutate_config(current_gateway_name: str, config: Config):
                         unique_key item_id,
                         batch_size 1
                     ),
+                    dialect {ctx.dialect},
                     {table_format}
                     start '2020-01-01',
                     end '2020-01-07',