feat: add DuckDB support with macro dispatches and CI configuration (#935)

Author: devin-ai-integration[bot]

.github/workflows/test-all-warehouses.yml

@@ -35,18 +35,20 @@ on:
         required: false
 
 jobs:
-  # ── Docker targets ────────────────────────────────────────────────────
+  # ── Local targets ─────────────────────────────────────────────────────
   # No secrets needed — run on pull_request (works for forks without approval).
   # Skipped on pull_request_target to avoid duplicate runs for internal PRs.
-  test-docker:
+  # Includes Docker-based adapters (postgres, clickhouse, trino, dremio) and
+  # fully in-process adapters (duckdb).
+  test-local:
     if: github.event_name != 'pull_request_target'
     strategy:
       fail-fast: false
       matrix:
         dbt-version:
           ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) ||
           fromJSON('["latest_official", "latest_pre"]') }}
-        warehouse-type: [postgres, clickhouse, trino, dremio]
+        warehouse-type: [postgres, clickhouse, trino, dremio, duckdb]
         exclude:
           # latest_pre is only tested on postgres
           - dbt-version: latest_pre
@@ -55,6 +57,8 @@ jobs:
             warehouse-type: trino
           - dbt-version: latest_pre
             warehouse-type: dremio
+          - dbt-version: latest_pre
+            warehouse-type: duckdb
     uses: ./.github/workflows/test-warehouse.yml
     with:
       warehouse-type: ${{ matrix.warehouse-type }}

.github/workflows/test-warehouse.yml

@@ -17,6 +17,7 @@ on:
           - trino
           - clickhouse
           - dremio
+          - duckdb
       elementary-ref:
         type: string
         required: false

integration_tests/dbt_project/macros/clear_env.sql

@@ -18,3 +18,8 @@
     {% do run_query("DROP DATABASE IF EXISTS " ~ schema_name) %}
     {% do adapter.commit() %}
 {% endmacro %}
+
+{% macro duckdb__edr_drop_schema(database_name, schema_name) %}
+    {% do run_query("DROP SCHEMA IF EXISTS " ~ schema_name ~ " CASCADE") %}
+    {% do adapter.commit() %}
+{% endmacro %}

integration_tests/profiles/profiles.yml.j2

@@ -2,7 +2,7 @@ elementary_tests:
   target: postgres
   outputs:
 
-    # ── Docker targets (plaintext, no secrets needed) ──────────────────
+    # ── Local targets (plaintext, no secrets needed) ─────────────────
 
     postgres: &postgres
       type: postgres
@@ -46,6 +46,12 @@ elementary_tests:
       schema: {{ schema_name }}
       threads: 4
 
+    duckdb: &duckdb
+      type: duckdb
+      path: ":memory:"
+      schema: {{ schema_name }}
+      threads: 8
+
     # ── Cloud targets (secrets substituted at CI time) ─────────────────
 
     snowflake: &snowflake
@@ -106,7 +112,7 @@ elementary_tests:
 elementary:
   target: postgres
   outputs:
-{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
+{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
 {%- for t in targets %}
     {{ t }}:
       <<: *{{ t }}

macros/edr/metadata_collection/get_columns_from_information_schema.sql

@@ -89,6 +89,21 @@
     {{ elementary.empty_table([('full_table_name', 'string'), ('database_name', 'string'), ('schema_name', 'string'), ('table_name', 'string'), ('column_name', 'string'), ('data_type', 'string')]) }}
 {% endmacro %}
 
+{% macro duckdb__get_columns_from_information_schema(database_name, schema_name, table_name = none) %}
+    select
+        upper(table_catalog || '.' || table_schema || '.' || table_name) as full_table_name,
+        upper(table_catalog) as database_name,
+        upper(table_schema) as schema_name,
+        upper(table_name) as table_name,
+        upper(column_name) as column_name,
+        data_type
+    from information_schema.columns
+    where upper(table_schema) = upper('{{ schema_name }}') and upper(table_catalog) = upper('{{ database_name }}')
+    {% if table_name %}
+      and upper(table_name) = upper('{{ table_name }}')
+    {% endif %}
+{% endmacro %}
+
 {% macro athena__get_columns_from_information_schema(database_name, schema_name, table_name = none) %}
     {{ elementary.empty_table([('full_table_name', 'string'), ('database_name', 'string'), ('schema_name', 'string'), ('table_name', 'string'), ('column_name', 'string'), ('data_type', 'string')]) }}
 {% endmacro %}

macros/edr/metadata_collection/get_tables_from_information_schema.sql

@@ -87,6 +87,29 @@
     from information_schema_tables
 {% endmacro %}
 
+{% macro duckdb__get_tables_from_information_schema(schema_tuple) %}
+    {%- set database_name, schema_name = schema_tuple %}
+
+    with information_schema_tables as (
+
+        select
+            upper(table_catalog) as database_name,
+            upper(table_schema) as schema_name,
+            upper(table_name) as table_name
+        from information_schema.tables
+            where upper(table_schema) = upper('{{ schema_name }}') and upper(table_catalog) = upper('{{ database_name }}')
+
+    )
+
+    select
+        {{ elementary.full_table_name() }} as full_table_name,
+        upper(database_name || '.' || schema_name) as full_schema_name,
+        database_name,
+        schema_name,
+        table_name
+    from information_schema_tables
+{% endmacro %}
+
 {% macro databricks__get_tables_from_information_schema(schema_tuple) %}
     {%- set database_name, schema_name = schema_tuple %}
     {% set schema_relation = api.Relation.create(database=database_name, schema=schema_name).without_identifier() %}

macros/edr/system/system_utils/buckets_cte.sql

@@ -144,6 +144,21 @@
     {{ return(complete_buckets_cte) }}
 {% endmacro %}
 
+{% macro duckdb__complete_buckets_cte(time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr) %}
+    {%- set complete_buckets_cte %}
+        select
+          unnest(generate_series({{ min_bucket_start_expr }}, {{ max_bucket_end_expr }}, interval '{{ time_bucket.count }} {{ time_bucket.period }}')) as edr_bucket_start
+    {%- endset %}
+    {%- set complete_buckets_cte %}
+        select
+          edr_bucket_start,
+          {{ bucket_end_expr }} as edr_bucket_end
+        from ({{ complete_buckets_cte }}) as _buckets
+        where {{ bucket_end_expr }} <= {{ max_bucket_end_expr }}
+    {%- endset %}
+    {{ return(complete_buckets_cte) }}
+{% endmacro %}
+
 {% macro dremio__complete_buckets_cte(time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr) %}
     {%- set complete_buckets_cte %}
         with integers as (

macros/edr/system/system_utils/full_names.sql

@@ -88,6 +88,20 @@
 {% endmacro %}
 
 
+{% macro duckdb__full_name_split(part_name) %}
+    {%- if part_name == 'database_name' -%}
+        {%- set part_index = 1 -%}
+    {%- elif part_name == 'schema_name' -%}
+        {%- set part_index = 2 -%}
+    {%- elif part_name == 'table_name' -%}
+        {%- set part_index = 3 -%}
+    {%- else -%}
+        {{ return('') }}
+    {%- endif -%}
+    trim(split_part(full_table_name,'.',{{ part_index }}),'"') as {{ part_name }}
+{% endmacro %}
+
+
 {% macro databricks__full_name_split(part_name) %}
     {%- if part_name == 'database_name' -%}
         {%- set part_index = 0 -%}
@@ -156,4 +170,4 @@
     {% set schemas_tuple = elementary.strings_list_to_tuple(schemas_list) %}
     {{ return(schemas_tuple) }}
 
-{% endmacro %}
+{% endmacro %}

macros/edr/tests/on_run_end/handle_tests_results.sql

@@ -132,6 +132,11 @@
     {%- do elementary.edr_create_table_as(true, temp_relation, test_tables_union_query) %}
     {% do elementary.run_query(insert_query) %}
 
+    {# DuckDB: commit so insert survives dbt's post-on-run-end ROLLBACK #}
+    {% if target.type == 'duckdb' %}
+        {% do adapter.commit() %}
+    {% endif %}
+
     {% if not elementary.has_temp_table_support() %}
         {% do elementary.fully_drop_relation(temp_relation) %}
     {% endif %}
@@ -181,6 +186,11 @@
     {%- do elementary.edr_create_table_as(true, temp_relation, test_tables_union_query) %}
     {% do elementary.run_query(insert_query) %}
 
+    {# DuckDB: commit so insert survives dbt's post-on-run-end ROLLBACK #}
+    {% if target.type == 'duckdb' %}
+        {% do adapter.commit() %}
+    {% endif %}
+
     {% if not elementary.has_temp_table_support() %}
         {% do elementary.fully_drop_relation(temp_relation) %}
     {% endif %}

macros/edr/tests/test_utils/clean_elementary_test_tables.sql

@@ -69,6 +69,10 @@
     {% do return(elementary.get_transactionless_clean_elementary_test_tables_queries(test_table_relations)) %}
 {% endmacro %}
 
+{% macro duckdb__get_clean_elementary_test_tables_queries(test_table_relations) %}
+    {% do return(elementary.get_transactionless_clean_elementary_test_tables_queries(test_table_relations)) %}
+{% endmacro %}
+
 {% macro dremio__get_clean_elementary_test_tables_queries(test_table_relations) %}
     {% do return(elementary.get_transactionless_clean_elementary_test_tables_queries(test_table_relations)) %}
 {% endmacro %}