ele-4409: added dbt group artifact and column

macros/edr/dbt_artifacts/upload_dbt_artifacts.sql

@@ -8,6 +8,7 @@
       "dbt_metrics": elementary.upload_dbt_metrics,
       "dbt_exposures": elementary.upload_dbt_exposures,
       "dbt_seeds": elementary.upload_dbt_seeds,
+      "dbt_groups": elementary.upload_dbt_groups,
       }
     %}
 

macros/edr/dbt_artifacts/upload_dbt_groups.sql

@@ -0,0 +1,36 @@
+{%- macro upload_dbt_groups(should_commit=false, metadata_hashes=none) -%}
+    {% set relation = elementary.get_elementary_relation('dbt_groups') %}
+    {% if execute and relation %}
+        {% set groups = graph.groups.values() | selectattr('resource_type', '==', 'group') %}
+        {% do elementary.upload_artifacts_to_table(relation, groups, elementary.flatten_group, should_commit=should_commit, metadata_hashes=metadata_hashes) %}
+    {%- endif -%}
+    {{- return('') -}}
+{%- endmacro -%}
+
+{% macro get_dbt_groups_empty_table_query() %}
+    {% set columns = [
+        ('unique_id', 'string'),
+        ('name', 'string'),
+        ('owner_email', 'string'),
+        ('owner_name', 'string'),
+        ('generated_at', 'string'),
+        ('metadata_hash', 'string'),
+    ] %}
+
+    {% set dbt_groups_empty_table_query = elementary.empty_table(columns) %}
+    {{ return(dbt_groups_empty_table_query) }}
+{% endmacro %}
+
+{% macro flatten_group(node_dict) %}
+    {% set owner_dict = elementary.safe_get_with_default(node_dict, 'owner', {}) %}
+
+    {% set flatten_group_metadata_dict = {
+        'unique_id': node_dict.get('unique_id'),
+        'name': node_dict.get('name'),
+        'owner_email': owner_dict.get('email'),
+        'owner_name': owner_dict.get('name'),
+        'generated_at': elementary.datetime_now_utc_as_string(),
+    } %}
+    {% do flatten_group_metadata_dict.update({'metadata_hash': elementary.get_artifact_metadata_hash(flatten_group_metadata_dict)}) %}
+    {{ return(flatten_group_metadata_dict) }}
+{% endmacro %}

macros/edr/dbt_artifacts/upload_dbt_metrics.sql

@@ -30,6 +30,7 @@
                                                                    ('path', 'string'),
                                                                    ('generated_at', 'string'),
                                                                    ('metadata_hash', 'string'),
+                                                                   ('group_name', 'string'),
                                                                    ]) %}
     {{ return(dbt_metrics_empty_table_query) }}
 {% endmacro %}
@@ -57,7 +58,8 @@
         'package_name': node_dict.get('package_name'),
         'original_path': node_dict.get('original_file_path'),
         'path': node_dict.get('path'),
-        'generated_at': elementary.datetime_now_utc_as_string()
+        'generated_at': elementary.datetime_now_utc_as_string(),
+        'group_name': config_dict.get("group"),
     }%}
     {% do flatten_metric_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_metric_metadata_dict)}) %}
     {{ return(flatten_metric_metadata_dict) }}

macros/edr/dbt_artifacts/upload_dbt_models.sql

@@ -30,6 +30,8 @@
         ('metadata_hash', 'string'),
         ('unique_key', 'string'),
         ('incremental_strategy', 'string'),
+        ('group_name', 'string'),
+        ('access', 'string'),
     ] %}
     {% if target.type == "bigquery" or elementary.get_config_var("include_other_warehouse_specific_columns") %}
         {% do columns.extend([('bigquery_partition_by', 'string'), ('bigquery_cluster_by', 'string')]) %}
@@ -62,7 +64,7 @@
     {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}
     {% set tags = elementary.union_lists(config_tags, global_tags) %}
     {% set tags = elementary.union_lists(tags, meta_tags) %}
-
+  
     {% set flatten_model_metadata_dict = {
         'unique_id': node_dict.get('unique_id'),
         'alias': node_dict.get('alias'),
@@ -86,6 +88,8 @@
         'incremental_strategy': config_dict.get("incremental_strategy"),
         'bigquery_partition_by': config_dict.get("partition_by"),
         'bigquery_cluster_by': config_dict.get("cluster_by"),
+        'group_name': config_dict.get("group") or node_dict.get("group"),
+        'access': config_dict.get("access") or node_dict.get("access"),
     } %}
     {% do flatten_model_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_model_metadata_dict)}) %}
     {{ return(flatten_model_metadata_dict) }}

macros/edr/dbt_artifacts/upload_dbt_seeds.sql

@@ -23,6 +23,7 @@
                                                                   ('path', 'string'),
                                                                   ('generated_at', 'string'),
                                                                   ('metadata_hash', 'string'),
+                                                                  ('group_name', 'string'),
                                                                   ]) %}
     {{ return(dbt_seeds_empty_table_query) }}
 {% endmacro %}
@@ -65,7 +66,8 @@
         'package_name': node_dict.get('package_name'),
         'original_path': node_dict.get('original_file_path'),
         'path': node_dict.get('path'),
-        'generated_at': elementary.datetime_now_utc_as_string()
+        'generated_at': elementary.datetime_now_utc_as_string(),
+        'group_name': config_dict.get("group") or node_dict.get("group"),
     }%}
     {% do flatten_seed_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_seed_metadata_dict)}) %}
     {{ return(flatten_seed_metadata_dict) }}

macros/edr/dbt_artifacts/upload_dbt_tests.sql

@@ -38,7 +38,8 @@
                                                                  ('path', 'string'),
                                                                  ('generated_at', 'string'),
                                                                  ('metadata_hash', 'string'),
-                                                                 ('quality_dimension', 'string')
+                                                                 ('quality_dimension', 'string'),
+                                                                 ('group_name', 'string'),
                                                                  ]) %}
     {{ return(dbt_tests_empty_table_query) }}
 {% endmacro %}
@@ -173,7 +174,8 @@
         'compiled_code': elementary.get_compiled_code(node_dict),
         'path': node_dict.get('path'),
         'generated_at': elementary.datetime_now_utc_as_string(),
-        'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_original_name, test_namespace)
+        'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_original_name, test_namespace),
+        'group_name': config_dict.get("group") or node_dict.get("group"),
     }%}
     {% do flatten_test_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_test_metadata_dict)}) %}
     {{ return(flatten_test_metadata_dict) }}

macros/edr/dbt_artifacts/upload_run_results.sql

@@ -31,7 +31,8 @@
                                                                        ('query_id', 'string'),
                                                                        ('thread_id', 'string'),
                                                                        ('materialization', 'string'),
-                                                                       ('adapter_response', 'string')
+                                                                       ('adapter_response', 'string'),
+                                                                       ('group_name', 'string'),
                                                                        ]) %}
     {{ return(dbt_run_results_empty_table_query) }}
 {% endmacro %}
@@ -61,7 +62,8 @@
         'query_id': run_result_dict.get('adapter_response', {}).get('query_id'),
         'thread_id': run_result_dict.get('thread_id'),
         'materialization': config_dict.get('materialized'),
-        'adapter_response': run_result_dict.get('adapter_response', {})
+        'adapter_response': run_result_dict.get('adapter_response', {}),
+        'group_name': config_dict.get('group'),
     } %}
 
     {% set timings = elementary.safe_get_with_default(run_result_dict, 'timing', []) %}

models/edr/dbt_artifacts/dbt_artifacts_hashes.sql

@@ -14,6 +14,7 @@
   "dbt_exposures",
   "dbt_seeds",
   "dbt_columns",
+  "dbt_groups",
 ] %}
 
 {% for artifact_model in artifact_models %}

models/edr/dbt_artifacts/dbt_groups.sql

@@ -0,0 +1,14 @@
+{{
+  config(
+    materialized='incremental',
+    transient=False,
+    post_hook='{{ elementary.upload_dbt_groups() }}',
+    unique_key='unique_id',
+    on_schema_change='sync_all_columns',
+    full_refresh=elementary.get_config_var('elementary_full_refresh'),
+    table_type=elementary.get_default_table_type(),
+    incremental_strategy=elementary.get_default_incremental_strategy()
+    )
+}}
+
+{{ elementary.get_dbt_groups_empty_table_query() }}

models/edr/run_results/model_run_results.sql

@@ -31,6 +31,7 @@ SELECT
     run_results.compiled_code,
     run_results.adapter_response,
     run_results.thread_id,
+    run_results.group_name,
     models.database_name,
     models.schema_name,
     coalesce(run_results.materialization, models.materialization) as materialization,

models/edr/run_results/seed_run_results.sql

@@ -31,6 +31,7 @@ SELECT
     run_results.compiled_code,
     run_results.adapter_response,
     run_results.thread_id,
+    run_results.group_name,
     seeds.database_name,
     seeds.schema_name,
     run_results.materialization,

models/edr/run_results/snapshot_run_results.sql

@@ -31,6 +31,7 @@ SELECT
     run_results.compiled_code,
     run_results.adapter_response,
     run_results.thread_id,
+    run_results.group_name,
     snapshots.database_name,
     snapshots.schema_name,
     coalesce(run_results.materialization, snapshots.materialization) as materialization,