Skip to content

Commit 59270ed

Browse files
arbivarbiv
authored
ele-4409: added dbt group artifact and column (#814)
* Added groups table and group column in relevant models and views * Added relevant Integration tests
1 parent 8c70686 commit 59270ed

13 files changed

Lines changed: 595 additions & 7 deletions

File tree

‎integration_tests/tests/test_dbt_artifacts/test_groups.py‎

Lines changed: 521 additions & 0 deletions
Large diffs are not rendered by default.

‎macros/edr/dbt_artifacts/upload_dbt_artifacts.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"dbt_metrics": elementary.upload_dbt_metrics,
99
"dbt_exposures": elementary.upload_dbt_exposures,
1010
"dbt_seeds": elementary.upload_dbt_seeds,
11+
"dbt_groups": elementary.upload_dbt_groups,
1112
}
1213
%}
1314

‎macros/edr/dbt_artifacts/upload_dbt_groups.sql‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{%- macro upload_dbt_groups(should_commit=false, metadata_hashes=none) -%}
2+
{% set relation = elementary.get_elementary_relation('dbt_groups') %}
3+
{% if execute and relation %}
4+
{% set groups = graph.groups.values() | selectattr('resource_type', '==', 'group') %}
5+
{% do elementary.upload_artifacts_to_table(relation, groups, elementary.flatten_group, should_commit=should_commit, metadata_hashes=metadata_hashes) %}
6+
{%- endif -%}
7+
{{- return('') -}}
8+
{%- endmacro -%}
9+
10+
{% macro get_dbt_groups_empty_table_query() %}
11+
{% set columns = [
12+
('unique_id', 'string'),
13+
('name', 'string'),
14+
('owner_email', 'string'),
15+
('owner_name', 'string'),
16+
('generated_at', 'string'),
17+
('metadata_hash', 'string'),
18+
] %}
19+
20+
{% set dbt_groups_empty_table_query = elementary.empty_table(columns) %}
21+
{{ return(dbt_groups_empty_table_query) }}
22+
{% endmacro %}
23+
24+
{% macro flatten_group(node_dict) %}
25+
{% set owner_dict = elementary.safe_get_with_default(node_dict, 'owner', {}) %}
26+
27+
{% set flatten_group_metadata_dict = {
28+
'unique_id': node_dict.get('unique_id'),
29+
'name': node_dict.get('name'),
30+
'owner_email': owner_dict.get('email'),
31+
'owner_name': owner_dict.get('name'),
32+
'generated_at': elementary.datetime_now_utc_as_string(),
33+
} %}
34+
{% do flatten_group_metadata_dict.update({'metadata_hash': elementary.get_artifact_metadata_hash(flatten_group_metadata_dict)}) %}
35+
{{ return(flatten_group_metadata_dict) }}
36+
{% endmacro %}

‎macros/edr/dbt_artifacts/upload_dbt_metrics.sql‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
('path', 'string'),
3131
('generated_at', 'string'),
3232
('metadata_hash', 'string'),
33+
('group_name', 'string'),
3334
]) %}
3435
{{ return(dbt_metrics_empty_table_query) }}
3536
{% endmacro %}
@@ -57,7 +58,8 @@
5758
'package_name': node_dict.get('package_name'),
5859
'original_path': node_dict.get('original_file_path'),
5960
'path': node_dict.get('path'),
60-
'generated_at': elementary.datetime_now_utc_as_string()
61+
'generated_at': elementary.datetime_now_utc_as_string(),
62+
'group_name': config_dict.get("group"),
6163
}%}
6264
{% do flatten_metric_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_metric_metadata_dict)}) %}
6365
{{ return(flatten_metric_metadata_dict) }}

‎macros/edr/dbt_artifacts/upload_dbt_models.sql‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
('metadata_hash', 'string'),
3131
('unique_key', 'string'),
3232
('incremental_strategy', 'string'),
33+
('group_name', 'string'),
34+
('access', 'string'),
3335
] %}
3436
{% if target.type == "bigquery" or elementary.get_config_var("include_other_warehouse_specific_columns") %}
3537
{% do columns.extend([('bigquery_partition_by', 'string'), ('bigquery_cluster_by', 'string')]) %}
@@ -62,7 +64,7 @@
6264
{% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}
6365
{% set tags = elementary.union_lists(config_tags, global_tags) %}
6466
{% set tags = elementary.union_lists(tags, meta_tags) %}
65-
67+
6668
{% set flatten_model_metadata_dict = {
6769
'unique_id': node_dict.get('unique_id'),
6870
'alias': node_dict.get('alias'),
@@ -86,6 +88,8 @@
8688
'incremental_strategy': config_dict.get("incremental_strategy"),
8789
'bigquery_partition_by': config_dict.get("partition_by"),
8890
'bigquery_cluster_by': config_dict.get("cluster_by"),
91+
'group_name': config_dict.get("group") or node_dict.get("group"),
92+
'access': config_dict.get("access") or node_dict.get("access"),
8993
} %}
9094
{% do flatten_model_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_model_metadata_dict)}) %}
9195
{{ return(flatten_model_metadata_dict) }}

‎macros/edr/dbt_artifacts/upload_dbt_seeds.sql‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
('path', 'string'),
2424
('generated_at', 'string'),
2525
('metadata_hash', 'string'),
26+
('group_name', 'string'),
2627
]) %}
2728
{{ return(dbt_seeds_empty_table_query) }}
2829
{% endmacro %}
@@ -65,7 +66,8 @@
6566
'package_name': node_dict.get('package_name'),
6667
'original_path': node_dict.get('original_file_path'),
6768
'path': node_dict.get('path'),
68-
'generated_at': elementary.datetime_now_utc_as_string()
69+
'generated_at': elementary.datetime_now_utc_as_string(),
70+
'group_name': config_dict.get("group") or node_dict.get("group"),
6971
}%}
7072
{% do flatten_seed_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_seed_metadata_dict)}) %}
7173
{{ return(flatten_seed_metadata_dict) }}

‎macros/edr/dbt_artifacts/upload_dbt_tests.sql‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@
3838
('path', 'string'),
3939
('generated_at', 'string'),
4040
('metadata_hash', 'string'),
41-
('quality_dimension', 'string')
41+
('quality_dimension', 'string'),
42+
('group_name', 'string'),
4243
]) %}
4344
{{ return(dbt_tests_empty_table_query) }}
4445
{% endmacro %}
@@ -173,7 +174,8 @@
173174
'compiled_code': elementary.get_compiled_code(node_dict),
174175
'path': node_dict.get('path'),
175176
'generated_at': elementary.datetime_now_utc_as_string(),
176-
'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_original_name, test_namespace)
177+
'quality_dimension': meta_dict.get('quality_dimension') or elementary.get_quality_dimension(test_original_name, test_namespace),
178+
'group_name': config_dict.get("group") or node_dict.get("group"),
177179
}%}
178180
{% do flatten_test_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_test_metadata_dict)}) %}
179181
{{ return(flatten_test_metadata_dict) }}

‎macros/edr/dbt_artifacts/upload_run_results.sql‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
('query_id', 'string'),
3232
('thread_id', 'string'),
3333
('materialization', 'string'),
34-
('adapter_response', 'string')
34+
('adapter_response', 'string'),
35+
('group_name', 'string'),
3536
]) %}
3637
{{ return(dbt_run_results_empty_table_query) }}
3738
{% endmacro %}
@@ -61,7 +62,8 @@
6162
'query_id': run_result_dict.get('adapter_response', {}).get('query_id'),
6263
'thread_id': run_result_dict.get('thread_id'),
6364
'materialization': config_dict.get('materialized'),
64-
'adapter_response': run_result_dict.get('adapter_response', {})
65+
'adapter_response': run_result_dict.get('adapter_response', {}),
66+
'group_name': config_dict.get('group'),
6567
} %}
6668

6769
{% set timings = elementary.safe_get_with_default(run_result_dict, 'timing', []) %}

‎models/edr/dbt_artifacts/dbt_artifacts_hashes.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"dbt_exposures",
1515
"dbt_seeds",
1616
"dbt_columns",
17+
"dbt_groups",
1718
] %}
1819

1920
{% for artifact_model in artifact_models %}

‎models/edr/dbt_artifacts/dbt_groups.sql‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{{
2+
config(
3+
materialized='incremental',
4+
transient=False,
5+
post_hook='{{ elementary.upload_dbt_groups() }}',
6+
unique_key='unique_id',
7+
on_schema_change='sync_all_columns',
8+
full_refresh=elementary.get_config_var('elementary_full_refresh'),
9+
table_type=elementary.get_default_table_type(),
10+
incremental_strategy=elementary.get_default_incremental_strategy()
11+
)
12+
}}
13+
14+
{{ elementary.get_dbt_groups_empty_table_query() }}

0 commit comments

Comments
 (0)