Merge pull request #443 from dlawin/issue_418_duckdb

adds a basic e2e test for --dbt using duckdb

Author: dlawin

.gitignore

@@ -74,6 +74,9 @@ docs/_build/
 # PyBuilder
 target/
 
+# Exception for dbt tests
+!tests/dbt_artifacts/target
+
 # Jupyter Notebook
 .ipynb_checkpoints
 

tests/dbt_artifacts/dbt_project.yml

@@ -0,0 +1,31 @@
+name: 'jaffle_shop'
+
+config-version: 2
+version: '0.1'
+
+profile: 'jaffle_shop'
+
+model-paths: ["models"]
+seed-paths: ["seeds"]
+test-paths: ["tests"]
+analysis-paths: ["analysis"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+    - "logs"
+
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+
+vars:
+  data_diff:
+    prod_database: jaffle_shop
+    prod_schema: prod
+
+models:
+  jaffle_shop:
+      materialized: table
+      staging:
+        materialized: view

tests/dbt_artifacts/jaffle_shop.duckdb

@@ -0,0 +1,7 @@
+jaffle_shop:
+  target: dev
+  outputs:
+    dev:
+      type: duckdb
+      path: "./tests/dbt_artifacts/jaffle_shop.duckdb"
+      schema: dev

tests/dbt_artifacts/profiles.yml

@@ -0,0 +1 @@
+{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v4.json", "dbt_version": "1.4.1", "generated_at": "2023-03-15T19:59:14.811639Z", "invocation_id": "ffabb889-60b0-4fcd-a4e6-456c549eb2c2", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2023-03-15T19:59:14.655050Z", "completed_at": "2023-03-15T19:59:14.657831Z"}, {"name": "execute", "started_at": "2023-03-15T19:59:14.658114Z", "completed_at": "2023-03-15T19:59:14.700161Z"}], "thread_id": "Thread-1 (worker)", "execution_time": 0.04775500297546387, "adapter_response": {"_message": "OK"}, "message": "OK", "failures": null, "unique_id": "model.jaffle_shop.stg_customers"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-03-15T19:59:14.705294Z", "completed_at": "2023-03-15T19:59:14.706915Z"}, {"name": "execute", "started_at": "2023-03-15T19:59:14.707179Z", "completed_at": "2023-03-15T19:59:14.720823Z"}], "thread_id": "Thread-1 (worker)", "execution_time": 0.01781010627746582, "adapter_response": {"_message": "OK"}, "message": "OK", "failures": null, "unique_id": "model.jaffle_shop.stg_orders"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-03-15T19:59:14.724207Z", "completed_at": "2023-03-15T19:59:14.725919Z"}, {"name": "execute", "started_at": "2023-03-15T19:59:14.726185Z", "completed_at": "2023-03-15T19:59:14.739744Z"}], "thread_id": "Thread-1 (worker)", "execution_time": 0.0189511775970459, "adapter_response": {"_message": "OK"}, "message": "OK", "failures": null, "unique_id": "model.jaffle_shop.stg_payments"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-03-15T19:59:14.746471Z", "completed_at": "2023-03-15T19:59:14.750253Z"}, {"name": "execute", "started_at": "2023-03-15T19:59:14.750460Z", "completed_at": "2023-03-15T19:59:14.776306Z"}], "thread_id": "Thread-1 (worker)", "execution_time": 0.03409934043884277, "adapter_response": {"_message": "OK"}, "message": "OK", "failures": null, "unique_id": "model.jaffle_shop.customers"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-03-15T19:59:14.781955Z", "completed_at": "2023-03-15T19:59:14.784545Z"}, {"name": "execute", "started_at": "2023-03-15T19:59:14.784765Z", "completed_at": "2023-03-15T19:59:14.799421Z"}], "thread_id": "Thread-1 (worker)", "execution_time": 0.021820068359375, "adapter_response": {"_message": "OK"}, "message": "OK", "failures": null, "unique_id": "model.jaffle_shop.orders"}], "elapsed_time": 0.20426106452941895, "args": {"write_json": true, "use_colors": true, "printer_width": 80, "version_check": true, "partial_parse": true, "static_parser": true, "profiles_dir": "/Users/dan/.dbt", "send_anonymous_usage_stats": true, "quiet": false, "no_print": false, "cache_selected_only": false, "which": "run", "rpc_method": "run", "indirect_selection": "eager"}}

tests/dbt_artifacts/target/manifest.json

@@ -326,16 +326,23 @@ def test_get_connection_no_type(self, mock_open):
 
 
 class TestDbtDiffer(unittest.TestCase):
-    # These two integration tests can be used to test a real diff
-    # export DATA_DIFF_DBT_PROJ=/path/to/a/dbt/project
-    # Expects a valid dbt project using a ~/.dbt/profiles.yml with run results
+    # Set DATA_DIFF_DBT_PROJ to use your own dbt project, otherwise uses the duckdb project in tests/dbt_artifacts
     def test_integration_basic_dbt(self):
-        project_dir = os.environ.get("DATA_DIFF_DBT_PROJ")
-        if project_dir is not None:
-            diff = run_datadiff_cli("--dbt", "--dbt-project-dir", project_dir)
-            assert diff[-1].decode("utf-8") == "Diffs Complete!"
-        else:
-            pass
+        artifacts_path = os.getcwd() + '/tests/dbt_artifacts'
+        test_project_path = os.environ.get("DATA_DIFF_DBT_PROJ") or artifacts_path
+        diff = run_datadiff_cli("--dbt", "--dbt-project-dir", test_project_path, "--dbt-profiles-dir", test_project_path)
+        assert diff[-1].decode("utf-8") == "Diffs Complete!"
+
+        # assertions for the diff that exists in tests/dbt_artifacts/jaffle_shop.duckdb
+        if test_project_path == artifacts_path:
+            diff_string = b''.join(diff).decode('utf-8')
+            # 5 diffs were ran
+            assert diff_string.count('<>') == 5
+            # 4 with no diffs
+            assert diff_string.count('No row differences') == 4
+            # 1 with a diff 
+            assert diff_string.count('| Rows Added    | Rows Removed') == 1
+
 
     def test_integration_cloud_dbt(self):
         project_dir = os.environ.get("DATA_DIFF_DBT_PROJ")