Skip to content

Commit 11677be

Browse files
haritamarclaude
andcommitted
feat: make project-scoped artifact filtering opt-in via config var
Add `upload_only_current_project_artifacts` config var (default: false) that, when enabled, filters graph entities by `package_name` so only artifacts from the current dbt project are uploaded -- excluding those from dependency packages. A shared `filter_to_current_project` helper macro centralizes the filtering logic and is called by all 7 upload_dbt_*.sql macros. Addresses PR #793 feedback from @elazarlachkar and @michael-myaskovsky. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent a4143a5 commit 11677be

File tree

9 files changed

+29
-14
lines changed

9 files changed

+29
-14
lines changed
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{%- macro filter_to_current_project(entities) -%}
2+
{%- if elementary.get_config_var("upload_only_current_project_artifacts") -%}
3+
{% set project_name = elementary.get_project_name() %}
4+
{% do return(
5+
entities | selectattr("package_name", "==", project_name) | list
6+
) %}
7+
{%- else -%} {% do return(entities | list) %}
8+
{%- endif -%}
9+
{%- endmacro -%}

macros/edr/dbt_artifacts/upload_dbt_columns.sql

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
{%- macro upload_dbt_columns(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_columns") %}
33
{% if execute and relation %}
4-
{% set tables = graph.nodes.values() | list + graph.sources.values() | list %}
4+
{% set tables = elementary.filter_to_current_project(
5+
graph.nodes.values()
6+
) + elementary.filter_to_current_project(graph.sources.values()) %}
57
{% do elementary.upload_artifacts_to_table(
68
relation,
79
tables,

macros/edr/dbt_artifacts/upload_dbt_exposures.sql

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{%- macro upload_dbt_exposures(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_exposures") %}
33
{% if execute and relation %}
4-
{% set exposures = graph.exposures.values() | selectattr(
5-
"resource_type", "==", "exposure"
4+
{% set exposures = elementary.filter_to_current_project(
5+
graph.exposures.values()
6+
| selectattr("resource_type", "==", "exposure")
67
) %}
78
{% do elementary.upload_artifacts_to_table(
89
relation,

macros/edr/dbt_artifacts/upload_dbt_models.sql

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{%- macro upload_dbt_models(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_models") %}
33
{% if execute and relation %}
4-
{% set models = graph.nodes.values() | selectattr(
5-
"resource_type", "==", "model"
4+
{% set models = elementary.filter_to_current_project(
5+
graph.nodes.values() | selectattr("resource_type", "==", "model")
66
) %}
77
{% do elementary.upload_artifacts_to_table(
88
relation,

macros/edr/dbt_artifacts/upload_dbt_seeds.sql

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{%- macro upload_dbt_seeds(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_seeds") %}
33
{% if execute and relation %}
4-
{% set seeds = graph.nodes.values() | selectattr(
5-
"resource_type", "==", "seed"
4+
{% set seeds = elementary.filter_to_current_project(
5+
graph.nodes.values() | selectattr("resource_type", "==", "seed")
66
) %}
77
{% do elementary.upload_artifacts_to_table(
88
relation,

macros/edr/dbt_artifacts/upload_dbt_snapshots.sql

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{%- macro upload_dbt_snapshots(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_snapshots") %}
33
{% if execute and relation %}
4-
{% set snapshots = graph.nodes.values() | selectattr(
5-
"resource_type", "==", "snapshot"
4+
{% set snapshots = elementary.filter_to_current_project(
5+
graph.nodes.values()
6+
| selectattr("resource_type", "==", "snapshot")
67
) %}
78
{% do elementary.upload_artifacts_to_table(
89
relation,

macros/edr/dbt_artifacts/upload_dbt_sources.sql

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{%- macro upload_dbt_sources(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_sources") %}
33
{% if execute and relation %}
4-
{% set sources = graph.sources.values() | selectattr(
5-
"resource_type", "==", "source"
4+
{% set sources = elementary.filter_to_current_project(
5+
graph.sources.values()
6+
| selectattr("resource_type", "==", "source")
67
) %}
78
{% do elementary.upload_artifacts_to_table(
89
relation,

macros/edr/dbt_artifacts/upload_dbt_tests.sql

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{%- macro upload_dbt_tests(should_commit=false, metadata_hashes=none) -%}
22
{% set relation = elementary.get_elementary_relation("dbt_tests") %}
33
{% if execute and relation %}
4-
{% set tests = graph.nodes.values() | selectattr(
5-
"resource_type", "==", "test"
4+
{% set tests = elementary.filter_to_current_project(
5+
graph.nodes.values() | selectattr("resource_type", "==", "test")
66
) %}
77
{% do elementary.upload_artifacts_to_table(
88
relation,
@@ -124,7 +124,7 @@
124124
{% set test_model_name_matches = modules.re.findall(
125125
"ref\(['\"](\w+)['\"]\)", test_model_jinja
126126
) + modules.re.findall(
127-
"source\(['\"]\w+['\"], ['\"](\w+)['\"]\)", test_model_jinja
127+
"source\(['\"]\w+['\"], ['\"](\ w+)['\"]\)", test_model_jinja
128128
) %}
129129
{% if test_model_name_matches | length == 1 %}
130130
{% set test_model_name = test_model_name_matches[0] %}

macros/edr/system/system_utils/get_config_var.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@
144144
"disable_samples_on_pii_tags": false,
145145
"pii_tags": ["pii"],
146146
"bigquery_disable_partitioning": false,
147+
"upload_only_current_project_artifacts": false,
147148
} %}
148149
{{- return(default_config) -}}
149150
{%- endmacro -%}

0 commit comments

Comments
 (0)