Chore: add external models to sushi project (#898)

* Chore: add external models to sushi project

* fix tests

Author: tobymao

examples/sushi/models/customers.sql

@@ -6,6 +6,26 @@ MODEL (
   tags (pii, fact)
 );
 
+CREATE SCHEMA IF NOT EXISTS raw;
+CREATE TABLE IF NOT EXISTS raw.marketing AS (
+  SELECT 1 AS customer_id, 'active' AS status
+);
+CREATE TABLE IF NOT EXISTS raw.demographics AS (
+  SELECT 1 AS customer_id, '00000' AS zip
+);
+
+WITH distinct_marketing AS (
+  SELECT DISTINCT
+    customer_id,
+    status
+  FROM raw.marketing
+)
 SELECT DISTINCT
-  customer_id::INT AS customer_id
+  o.customer_id::INT AS customer_id,
+  COALESCE(m.status, 'UNKNOWN') AS status,
+  d.zip
 FROM sushi.orders AS o
+LEFT JOIN distinct_marketing AS m
+    ON o.customer_id = m.customer_id
+LEFT JOIN raw.demographics AS d
+    ON o.customer_id = d.customer_id

examples/sushi/schema.yaml

@@ -0,0 +1,10 @@
+- name: raw.marketing
+  description: Table containing marketing information
+  columns:
+    customer_id: int
+    status: text
+- name: raw.demographics
+  description: Table containing demographics information
+  columns:
+    customer_id: int
+    zip: text

tests/conftest.py

@@ -31,11 +31,6 @@ def ignore_local_config_files():
         yield
 
 
-@pytest.fixture
-def context(tmpdir) -> Context:
-    return Context(paths=str(tmpdir))
-
-
 @pytest.fixture
 def duck_conn() -> duckdb.DuckDBPyConnection:
     return duckdb.connect()

tests/core/test_integration.py

@@ -798,10 +798,10 @@ def validate_environment_views(
 ) -> None:
     adapter = context.engine_adapter
     for snapshot in snapshots:
-        if snapshot.is_embedded:
+        if snapshot.is_symbolic:
             continue
-
         view_name = snapshot.qualified_view_name.for_environment(environment=environment)
+
         assert adapter.table_exists(view_name)
         assert select_all(
             snapshot.table_name(is_dev=environment != c.PROD, for_read=True), adapter

tests/core/test_schema_loader.py

@@ -1,30 +1,26 @@
-import os
-
 import pandas as pd
-import pytest
 from sqlglot import exp
 
-from sqlmesh.core import constants as c
+from sqlmesh.core.config import Config, DuckDBConnectionConfig, GatewayConfig
+from sqlmesh.core.context import Context
 from sqlmesh.core.dialect import parse
 from sqlmesh.core.model import load_model
 from sqlmesh.core.snapshot import SnapshotChangeCategory
 
 
-@pytest.fixture(autouse=True)
-def cleanup(sushi_context):
-    yield
-    os.remove(sushi_context.path / c.SCHEMA_YAML)
-
+def test_create_external_models(tmpdir, assert_exp_eq):
+    config = Config(gateways=GatewayConfig(connection=DuckDBConnectionConfig()))
+    context = Context(paths=[tmpdir], config=config)
 
-def test_create_external_models(sushi_context, assert_exp_eq):
     fruits = pd.DataFrame(
         [
             {"id": 1, "name": "apple"},
             {"id": 2, "name": "banana"},
         ]
     )
 
-    cursor = sushi_context.engine_adapter.cursor
+    cursor = context.engine_adapter.cursor
+    cursor.execute("CREATE SCHEMA sushi")
     cursor.execute("CREATE TABLE sushi.raw_fruits AS SELECT * FROM fruits")
 
     model = load_model(
@@ -40,12 +36,12 @@ def test_create_external_models(sushi_context, assert_exp_eq):
         )
     )
 
-    sushi_context.upsert_model(model)
-    sushi_context.create_external_models()
-    assert sushi_context.models["sushi.fruits"].columns_to_types == {
+    context.upsert_model(model)
+    context.create_external_models()
+    assert context.models["sushi.fruits"].columns_to_types == {
         "name": exp.DataType.build("UNKNOWN")
     }
-    sushi_context.load()
+    context.load()
 
     model = load_model(
         parse(
@@ -60,27 +56,27 @@ def test_create_external_models(sushi_context, assert_exp_eq):
         )
     )
 
-    sushi_context.upsert_model(model)
-    raw_fruits = sushi_context.models["sushi.raw_fruits"]
+    context.upsert_model(model)
+    raw_fruits = context.models["sushi.raw_fruits"]
     assert raw_fruits.kind.is_symbolic
     assert raw_fruits.kind.is_external
     assert raw_fruits.columns_to_types == {
         "id": exp.DataType.build("BIGINT"),
         "name": exp.DataType.build("VARCHAR"),
     }
 
-    snapshot = sushi_context.snapshots["sushi.raw_fruits"]
+    snapshot = context.snapshots["sushi.raw_fruits"]
     snapshot.categorize_as(SnapshotChangeCategory.BREAKING)
 
-    fruits = sushi_context.models["sushi.fruits"]
+    fruits = context.models["sushi.fruits"]
     assert not fruits.kind.is_symbolic
     assert not fruits.kind.is_external
     assert fruits.columns_to_types == {
         "id": exp.DataType.build("BIGINT"),
         "name": exp.DataType.build("VARCHAR"),
     }
     assert_exp_eq(
-        fruits.render_query(snapshots=sushi_context.snapshots),
+        fruits.render_query(snapshots=context.snapshots),
         """
         SELECT
           raw_fruits.id AS id,

tests/integrations/github/cicd/test_github_controller.py

@@ -121,10 +121,11 @@ def test_delete_pr_environment(
     controller = github_pr_synchronized_approvers_controller
     controller._context._engine_adapter = mock_engine_adapter = mocker.MagicMock()
     controller.delete_pr_environment()
-    assert mock_engine_adapter.method_calls == [
+    assert sorted(mock_engine_adapter.method_calls, key=str) == [
+        call.drop_schema(schema_name="raw__hello_world_2", ignore_if_not_exists=True, cascade=True),
         call.drop_schema(
             schema_name="sushi__hello_world_2", ignore_if_not_exists=True, cascade=True
-        )
+        ),
     ]