From 5552ab2168261c693062baf9ecbd7d057f1828a9 Mon Sep 17 00:00:00 2001 From: Padraig Alton Date: Fri, 9 Feb 2024 17:05:50 +0000 Subject: [PATCH 1/2] Add support for ephemeral models --- CHANGELOG.md | 4 ++++ dbt2looker/models.py | 5 +++-- dbt2looker/parser.py | 10 +++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9943468..676d862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Recent and upcoming changes to dbt2looker +## Unreleased +### Added +- support ephemeral models (#57) + ## 0.11.0 ### Added - support label and hidden fields (#49) diff --git a/dbt2looker/models.py b/dbt2looker/models.py index 3430eaf..9961182 100644 --- a/dbt2looker/models.py +++ b/dbt2looker/models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Union, Dict, List, Optional +from typing import Any, Union, Dict, List, Optional try: from typing import Literal except ImportError: @@ -144,6 +144,7 @@ class DbtModelColumn(BaseModel): class DbtNode(BaseModel): unique_id: str resource_type: str + config: Dict[str, Any] class Dbt2LookerExploreJoin(BaseModel): @@ -224,4 +225,4 @@ def case_insensitive_column_names(cls, v: Dict[str, DbtCatalogNodeColumn]): class DbtCatalog(BaseModel): - nodes: Dict[str, DbtCatalogNode] \ No newline at end of file + nodes: Dict[str, DbtCatalogNode] diff --git a/dbt2looker/parser.py b/dbt2looker/parser.py index ed310f3..03045c9 100644 --- a/dbt2looker/parser.py +++ b/dbt2looker/parser.py @@ -31,21 +31,21 @@ def tags_match(query_tag: str, model: models.DbtModel) -> bool: def parse_models(raw_manifest: dict, tag=None) -> List[models.DbtModel]: manifest = models.DbtManifest(**raw_manifest) - all_models: List[models.DbtModel] = [ + materialized_models: List[models.DbtModel] = [ node for node in manifest.nodes.values() - if node.resource_type == 'model' + if node.resource_type == 'model' and node.config['materialized'] != 'ephemeral' ] # Empty model files have many missing parameters - for model in all_models: + for model in materialized_models: if not hasattr(model, 'name'): logging.error('Cannot parse model with id: "%s" - is the model file empty?', model.unique_id) raise SystemExit('Failed') if tag is None: - return all_models - return [model for model in all_models if tags_match(tag, model)] + return materialized_models + return [model for model in materialized_models if tags_match(tag, model)] def check_models_for_missing_column_types(dbt_typed_models: List[models.DbtModel]): From 8aad35994af892777a60194ec7a88053c25c57bc Mon Sep 17 00:00:00 2001 From: Padraig Alton Date: Fri, 9 Feb 2024 17:14:27 +0000 Subject: [PATCH 2/2] Reorder parse_models logic to ignore non-selected empty model files --- CHANGELOG.md | 3 +++ dbt2looker/parser.py | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 676d862..7066f58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Recent and upcoming changes to dbt2looker ### Added - support ephemeral models (#57) +### Changed +- only non-ephemeral models _selected by tag logic_ are checked to ensure the model files are not empty (instead of all models) (#57) + ## 0.11.0 ### Added - support label and hidden fields (#49) diff --git a/dbt2looker/parser.py b/dbt2looker/parser.py index 03045c9..f49477f 100644 --- a/dbt2looker/parser.py +++ b/dbt2looker/parser.py @@ -37,15 +37,18 @@ def parse_models(raw_manifest: dict, tag=None) -> List[models.DbtModel]: if node.resource_type == 'model' and node.config['materialized'] != 'ephemeral' ] + if tag is None: + selected_models = materialized_models + else: + selected_models = [model for model in materialized_models if tags_match(tag, model)] + # Empty model files have many missing parameters - for model in materialized_models: + for model in selected_models: if not hasattr(model, 'name'): logging.error('Cannot parse model with id: "%s" - is the model file empty?', model.unique_id) raise SystemExit('Failed') - if tag is None: - return materialized_models - return [model for model in materialized_models if tags_match(tag, model)] + return selected_models def check_models_for_missing_column_types(dbt_typed_models: List[models.DbtModel]):