Merge branch 'TobikoData:main' into doris

Author: xinge-ji

.circleci/continue_config.yml

@@ -239,7 +239,7 @@ jobs:
       - checkout
       - run:
           name: Install OS-level dependencies
-          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"      
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
       - run:
           name: Generate database name
           command: |
@@ -308,7 +308,7 @@ workflows:
                 - redshift
                 - bigquery
                 - clickhouse-cloud
-                - athena                
+                - athena
                 - fabric
                 - gcp-postgres
           filters:

.circleci/manage-test-db.sh

@@ -51,7 +51,7 @@ databricks_init() {
 
     # Note: the cluster doesnt need to be running to create / drop catalogs, but it does need to be running to run the integration tests
     echo "Ensuring cluster is running"
-    databricks clusters start $CLUSTER_ID || true
+    databricks clusters start $CLUSTER_ID
 }
 
 databricks_up() {
@@ -80,7 +80,11 @@ redshift_down() {
     EXIT_CODE=1
     ATTEMPTS=0
     while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
-        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'"
+        # note: sometimes this pg_terminate_backend() call can randomly fail with: ERROR:  Insufficient privileges 
+        # if it does, let's proceed with the drop anyway rather than aborting and never attempting the drop
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'" || true
+        
+        # perform drop
         redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
         if [ $EXIT_CODE -ne 0 ]; then
             echo "Unable to drop database; retrying..."

.github/workflows/pr.yaml

@@ -8,6 +8,8 @@ on:
 concurrency:
   group: 'pr-${{ github.event.pull_request.number }}'
   cancel-in-progress: true
+permissions:
+  contents: read
 jobs:
   test-vscode:
     env:
@@ -56,7 +58,7 @@ jobs:
         run: pnpm exec playwright install
       - name: Run e2e tests
         working-directory: ./vscode/extension
-        timeout-minutes: 90
+        timeout-minutes: 30
         run: |
           source ../../.venv/bin/activate
           pnpm run test:e2e
@@ -66,3 +68,79 @@ jobs:
           name: playwright-report
           path: vscode/extension/playwright-report/
           retention-days: 30
+  test-dbt-versions:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        dbt-version: ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10']
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install SQLMesh dev dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          UV=1 make install-dev-dbt-${{ matrix.dbt-version }}
+      - name: Run dbt tests
+        # We can't run slow tests across all engines due to tests requiring DuckDB and old versions
+        # of DuckDB require a version of DuckDB we no longer support
+        run: |
+          source .venv/bin/activate
+
+          # Remove semantic_models and metrics sections for DBT versions < 1.6.0
+          # Using explicit list to avoid version comparison issues
+          if [[ "${{ matrix.dbt-version }}" == "1.3" ]] || \
+             [[ "${{ matrix.dbt-version }}" == "1.4" ]] || \
+             [[ "${{ matrix.dbt-version }}" == "1.5" ]]; then
+            
+            echo "DBT version is ${{ matrix.dbt-version }} (< 1.6.0), removing semantic_models and metrics sections..."
+            
+            schema_file="tests/fixtures/dbt/sushi_test/models/schema.yml"
+            if [[ -f "$schema_file" ]]; then
+              echo "Modifying $schema_file..."
+              
+              # Create a temporary file
+              temp_file=$(mktemp)
+              
+              # Use awk to remove semantic_models and metrics sections
+              awk '
+                /^semantic_models:/ { in_semantic=1; next }
+                /^metrics:/ { in_metrics=1; next }
+                /^[^ ]/ && (in_semantic || in_metrics) { 
+                  in_semantic=0; 
+                  in_metrics=0 
+                }
+                !in_semantic && !in_metrics { print }
+              ' "$schema_file" > "$temp_file"
+              
+              # Move the temp file back
+              mv "$temp_file" "$schema_file"
+              
+              echo "Successfully removed semantic_models and metrics sections"
+            else
+              echo "Schema file not found at $schema_file, skipping..."
+            fi
+          else
+            echo "DBT version is ${{ matrix.dbt-version }} (>= 1.6.0), keeping semantic_models and metrics sections"
+          fi
+
+          make dbt-fast-test
+      - name: Test SQLMesh info in sushi_dbt
+        working-directory: ./examples/sushi_dbt
+        run: |
+          source ../../.venv/bin/activate
+          sed -i 's/target: in_memory/target: postgres/g' profiles.yml
+          if [[ $(echo -e "${{ matrix.dbt-version }}\n1.5.0" | sort -V | head -n1) == "${{ matrix.dbt-version }}" ]] && [[ "${{ matrix.dbt-version }}" != "1.5.0" ]]; then
+            echo "DBT version is ${{ matrix.dbt-version }} (< 1.5.0), removing version parameters..."
+            sed -i -e 's/, version=1) }}/) }}/g' -e 's/, v=1) }}/) }}/g' models/top_waiters.sql
+          else
+            echo "DBT version is ${{ matrix.dbt-version }} (>= 1.5.0), keeping version parameters"
+          fi
+
+          sqlmesh info --skip-connection

Makefile

@@ -1,14 +1,58 @@
 .PHONY: docs
 
+ifdef UV
+    PIP := uv pip
+else
+    PIP := pip3
+endif
+
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Darwin)
+    SED_INPLACE = sed -i ''
+else
+    SED_INPLACE = sed -i
+endif
+
 install-dev:
-	pip3 install -e ".[dev,web,slack,dlt,lsp]" ./examples/custom_materializations
+	$(PIP) install -e ".[dev,web,slack,dlt,lsp]" ./examples/custom_materializations
 
 install-doc:
-	pip3 install -r ./docs/requirements.txt
+	$(PIP) install -r ./docs/requirements.txt
 
 install-pre-commit:
 	pre-commit install
 
+install-dev-dbt-%:
+	@version="$*"; \
+	period_count=$$(echo "$$version" | tr -cd '.' | wc -c); \
+	if [ "$$period_count" -eq 0 ]; then \
+		version="$${version:0:1}.$${version:1}"; \
+	elif [ "$$period_count" -eq 1 ]; then \
+		version="$$version.0"; \
+	fi; \
+	echo "Installing dbt version: $$version"; \
+	cp pyproject.toml pyproject.toml.backup; \
+	$(SED_INPLACE) 's/"pydantic>=2.0.0"/"pydantic"/g' pyproject.toml; \
+	if [ "$$version" = "1.10.0" ]; then \
+		echo "Applying special handling for dbt 1.10.0"; \
+		$(SED_INPLACE) -E 's/"(dbt-core)[^"]*"/"\1~='"$$version"'"/g' pyproject.toml; \
+		$(SED_INPLACE) -E 's/"(dbt-(bigquery|duckdb|snowflake|athena-community|clickhouse|databricks|redshift|trino))[^"]*"/"\1"/g' pyproject.toml; \
+	else \
+		echo "Applying version $$version to all dbt packages"; \
+		$(SED_INPLACE) -E 's/"(dbt-[^"><=~!]+)[^"]*"/"\1~='"$$version"'"/g' pyproject.toml; \
+	fi; \
+	$(MAKE) install-dev; \
+	if [ "$$version" = "1.6.0" ]; then \
+		echo "Applying overrides for dbt 1.6.0"; \
+		$(PIP) install 'pydantic>=2.0.0' 'google-cloud-bigquery==3.30.0' 'databricks-sdk==0.28.0' --reinstall; \
+	fi; \
+	if [ "$$version" = "1.7.0" ]; then \
+		echo "Applying overrides for dbt 1.7.0"; \
+		$(PIP) install 'databricks-sdk==0.28.0' --reinstall; \
+	fi; \
+	mv pyproject.toml.backup pyproject.toml; \
+	echo "Restored original pyproject.toml"
+
 style:
 	pre-commit run --all-files
 
@@ -22,16 +66,16 @@ doc-test:
 	python -m pytest --doctest-modules sqlmesh/core sqlmesh/utils
 
 package:
-	pip3 install build && python3 -m build
+	$(PIP) install build && python3 -m build
 
 publish: package
-	pip3 install twine && python3 -m twine upload dist/*
+	$(PIP) install twine && python3 -m twine upload dist/*
 
 package-tests:
-	pip3 install build && cp pyproject.toml tests/sqlmesh_pyproject.toml && python3 -m build tests/
+	$(PIP) install build && cp pyproject.toml tests/sqlmesh_pyproject.toml && python3 -m build tests/
 
 publish-tests: package-tests
-	pip3 install twine && python3 -m twine upload -r tobiko-private tests/dist/*
+	$(PIP) install twine && python3 -m twine upload -r tobiko-private tests/dist/*
 
 docs-serve:
 	mkdocs serve
@@ -93,6 +137,9 @@ engine-test:
 dbt-test:
 	pytest -n auto -m "dbt and not cicdonly"
 
+dbt-fast-test:
+	pytest -n auto -m "dbt and fast" --retries 3
+
 github-test:
 	pytest -n auto -m "github"
 
@@ -109,7 +156,7 @@ guard-%:
 	fi
 
 engine-%-install:
-	pip3 install -e ".[dev,web,slack,lsp,${*}]" ./examples/custom_materializations
+	$(PIP) install -e ".[dev,web,slack,lsp,${*}]" ./examples/custom_materializations
 
 engine-docker-%-up:
 	docker compose -f ./tests/core/engine_adapter/integration/docker/compose.${*}.yaml up -d
@@ -159,11 +206,11 @@ snowflake-test: guard-SNOWFLAKE_ACCOUNT guard-SNOWFLAKE_WAREHOUSE guard-SNOWFLAK
 	pytest -n auto -m "snowflake" --retries 3 --junitxml=test-results/junit-snowflake.xml
 
 bigquery-test: guard-BIGQUERY_KEYFILE engine-bigquery-install
-	pip install -e ".[bigframes]"
+	$(PIP) install -e ".[bigframes]"
 	pytest -n auto -m "bigquery" --retries 3 --junitxml=test-results/junit-bigquery.xml
 
 databricks-test: guard-DATABRICKS_CATALOG guard-DATABRICKS_SERVER_HOSTNAME guard-DATABRICKS_HTTP_PATH guard-DATABRICKS_ACCESS_TOKEN guard-DATABRICKS_CONNECT_VERSION engine-databricks-install
-	pip install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}'
+	$(PIP) install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}'
 	pytest -n auto -m "databricks" --retries 3 --junitxml=test-results/junit-databricks.xml
 
 redshift-test: guard-REDSHIFT_HOST guard-REDSHIFT_USER guard-REDSHIFT_PASSWORD guard-REDSHIFT_DATABASE engine-redshift-install
@@ -176,7 +223,7 @@ athena-test: guard-AWS_ACCESS_KEY_ID guard-AWS_SECRET_ACCESS_KEY guard-ATHENA_S3
 	pytest -n auto -m "athena" --retries 3 --junitxml=test-results/junit-athena.xml
 
 fabric-test: guard-FABRIC_HOST guard-FABRIC_CLIENT_ID guard-FABRIC_CLIENT_SECRET guard-FABRIC_DATABASE engine-fabric-install
-	pytest -n auto -m "fabric" --retries 3 --junitxml=test-results/junit-fabric.xml	
+	pytest -n auto -m "fabric" --retries 3 --junitxml=test-results/junit-fabric.xml
 
 gcp-postgres-test: guard-GCP_POSTGRES_INSTANCE_CONNECTION_STRING guard-GCP_POSTGRES_USER guard-GCP_POSTGRES_PASSWORD guard-GCP_POSTGRES_KEYFILE_JSON engine-gcppostgres-install
 	pytest -n auto -m "gcp_postgres" --retries 3 --junitxml=test-results/junit-gcp-postgres.xml

docs/integrations/engines/snowflake.md

@@ -250,6 +250,14 @@ And confirm that our schemas and objects exist in the Snowflake catalog:
 
 Congratulations - your SQLMesh project is up and running on Snowflake!
 
+### Where are the row counts?
+
+SQLMesh reports the number of rows processed by each model in its `plan` and `run` terminal output.
+
+However, due to limitations in the Snowflake Python connector, row counts cannot be determined for `CREATE TABLE AS` statements. Therefore, SQLMesh does not report row counts for certain model kinds, such as `FULL` models.
+
+Learn more about the connector limitation [on Github](https://github.com/snowflakedb/snowflake-connector-python/issues/645).
+
 ## Local/Built-in Scheduler
 **Engine Adapter Type**: `snowflake`
 

examples/sushi/models/customer_revenue_by_day.sql

@@ -21,7 +21,7 @@ WITH order_total AS (
   LEFT JOIN sushi.items AS i
     ON oi.item_id = i.id AND oi.event_date = i.event_date
   WHERE
-    oi.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND CAST('{{ end_ds }}' as DATE)
+    oi.event_date BETWEEN @start_date AND @end_date
   GROUP BY
     oi.order_id,
     oi.event_date
@@ -35,7 +35,7 @@ FROM sushi.orders AS o
 LEFT JOIN order_total AS ot
   ON o.id = ot.order_id AND o.event_date = ot.event_date
 WHERE
-  o.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND CAST('{{ end_ds }}' as DATE)
+  o.event_date BETWEEN @start_date AND @end_date
 GROUP BY
   o.customer_id,
   o.event_date

examples/sushi/models/waiter_as_customer_by_day.sql

@@ -27,6 +27,6 @@ SELECT
 FROM sushi.waiters AS w
 JOIN sushi.customers as c ON w.waiter_id = c.customer_id
 JOIN sushi.waiter_names as wn ON w.waiter_id = wn.id
-WHERE w.event_date BETWEEN @start_date AND @end_date;
+WHERE w.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND @end_date;
 
 JINJA_END;

examples/sushi_dbt/models/schema.yml

@@ -21,6 +21,8 @@ models:
         tests:
           - less_than_amount:
               amount: 1000
+          - greater_than_amount:
+              amount: 0
       - name: ds
         description: Date
   - name: top_waiters
@@ -34,6 +36,8 @@ models:
               field: waiter_id
       - name: revenue
         description: Revenue from orders served by this waiter
+      - name: unused_column
+        data_type: int
   - name: waiters
     columns:
       - name: waiter_id

examples/sushi_dbt/models/top_waiters.sql

@@ -6,7 +6,8 @@
 
 SELECT
   waiter_id::INT AS waiter_id,
-  revenue::DOUBLE AS revenue
+  revenue::DOUBLE AS revenue,
+  1 AS unused_column
 FROM {{ ref('waiter_revenue_by_day', version=1) }}
 WHERE
   ds = (

examples/sushi_dbt/profiles.yml

@@ -3,6 +3,14 @@ sushi:
     in_memory:
       type: duckdb
       schema: sushi
+    postgres:
+      type: postgres
+      host: "host"
+      user: "user"
+      password: "password"
+      dbname: "dbname"
+      port: 5432
+      schema: sushi
     duckdb:
       type: duckdb
       path: 'local.duckdb'