diff --git a/src/dwh2looker/lookml_generator/generators.py b/src/dwh2looker/lookml_generator/generators.py index 208365e..6a7dd0f 100644 --- a/src/dwh2looker/lookml_generator/generators.py +++ b/src/dwh2looker/lookml_generator/generators.py @@ -76,13 +76,14 @@ def _get_looker_type(self, field: db.Field): # Default unknown types to string return FIELD_TYPE_MAPPING[self.db_type].get(field.internal_type, "string") - def create_dimension(self, field: db.Field): + def create_dimension(self, field: db.Field, parent_view_path: str = None): + original_field_name = field.name field_name = field.name is_array = field.is_array_field is_nested = field.is_nested_field field_description = field.description field_type = field.internal_type - field_sql_name = f"${{TABLE}}.{field.name}" + field_sql_name = f"${{TABLE}}.{original_field_name}" lookml_type = self._get_looker_type(field) pk = None convert_ts = None @@ -109,13 +110,35 @@ def create_dimension(self, field: db.Field): if field.parent_field_name: if field.parent_field_type == "RECORD_NULLABLE": - field_sql_name = f"${{TABLE}}.{field.parent_field_name}.{field_name}" + field_sql_name = ( + f"${{TABLE}}.{field.parent_field_name}.{original_field_name}" + ) field_name = ( f"{field.parent_field_name.replace('.', '__')}__{field_name}" ) elif field.parent_field_type == "ARRAY": field_sql_name = "${TABLE}" + # If we are in an unnested view, we need to make the SQL relative to ${TABLE} + if parent_view_path and "." in parent_view_path: + parts = parent_view_path.split(".") + array_path = ".".join(parts[1:]) + + if field.parent_field_name and field.parent_field_name.startswith( + array_path + ): + relative_parent_path = field.parent_field_name[ + len(array_path) : + ].lstrip(".") + if relative_parent_path: + field_sql_name = ( + f"${{TABLE}}.{relative_parent_path}.{original_field_name}" + ) + else: + field_sql_name = f"${{TABLE}}.{original_field_name}" + elif field.parent_field_name == array_path: + field_sql_name = f"${{TABLE}}.{original_field_name}" + if ( not (is_nested or is_array) and field.parent_field_type not in ["ARRAY"] @@ -211,9 +234,10 @@ def _build_timeframes(self, field_type: str): else: return self.timeframes - def create_dimension_group(self, field: db.Field): + def create_dimension_group(self, field: db.Field, parent_view_path: str = None): + original_field_name = field.name field_name = field.name - field_sql_name = f"${{TABLE}}.{field.name}" + field_sql_name = f"${{TABLE}}.{original_field_name}" field_type = field.internal_type field_description = field.description hidden = None @@ -237,7 +261,34 @@ def create_dimension_group(self, field: db.Field): ) if field.parent_field_name: - field_sql_name = f"${{TABLE}}.{field.name}" + if field.parent_field_type == "RECORD_NULLABLE": + field_sql_name = ( + f"${{TABLE}}.{field.parent_field_name}.{original_field_name}" + ) + field_name = ( + f"{field.parent_field_name.replace('.', '__')}__{field_name}" + ) + + # If we are in an unnested view, we need to make the SQL relative to ${TABLE} + if parent_view_path and "." in parent_view_path: + parts = parent_view_path.split(".") + array_path = ".".join(parts[1:]) + + if field.parent_field_name and field.parent_field_name.startswith( + array_path + ): + relative_parent_path = field.parent_field_name[ + len(array_path) : + ].lstrip(".") + if relative_parent_path: + field_sql_name = ( + f"${{TABLE}}.{relative_parent_path}.{original_field_name}" + ) + else: + field_sql_name = f"${{TABLE}}.{original_field_name}" + elif field.parent_field_name == array_path: + field_sql_name = f"${{TABLE}}.{original_field_name}" + group_item_label = self.nested_field_helper.build_field_name(field.name) group_label = self.nested_field_helper.build_field_name( field.parent_field_name diff --git a/src/dwh2looker/lookml_generator/lookml_generator.py b/src/dwh2looker/lookml_generator/lookml_generator.py index 5f0fffe..873bcb3 100644 --- a/src/dwh2looker/lookml_generator/lookml_generator.py +++ b/src/dwh2looker/lookml_generator/lookml_generator.py @@ -182,7 +182,7 @@ def sort_fields(self, fields: list[db.Field]) -> list[db.Field]: + nested_fields ) - def process_field(self, field: db.Field): + def process_field(self, field: db.Field, parent_view_path: str = None): field_mode = field.mode field_type = field.internal_type @@ -199,12 +199,14 @@ def process_field(self, field: db.Field): ] ): dimension_group = self.dimension_group_generator.create_dimension_group( - field + field, parent_view_path=parent_view_path ) yield self.dimension_group_generator.render(dimension_group) else: - dimension = self.dimension_generator.create_dimension(field) + dimension = self.dimension_generator.create_dimension( + field, parent_view_path=parent_view_path + ) yield self.dimension_generator.render(dimension) def process_views( @@ -252,7 +254,9 @@ def process_views( processed_fields = [] for field in fields_to_process_for_view: if not field.is_struct_field: - processed_field = self.process_field(field) + processed_field = self.process_field( + field, parent_view_path=current_view_path + ) if processed_field: processed_fields.extend(processed_field) diff --git a/src/tests/test_dimension_group_issue.py b/src/tests/test_dimension_group_issue.py new file mode 100644 index 0000000..e1f82d5 --- /dev/null +++ b/src/tests/test_dimension_group_issue.py @@ -0,0 +1,49 @@ +from unittest.mock import Mock +import pytest +from dwh2looker.db_client.db_client import Field +from dwh2looker.lookml_generator.generators import ( + DimensionGroupGenerator, + NestedFieldHelper, +) + + +@pytest.fixture +def jinja_env(): + return Mock() + + +@pytest.fixture +def nested_field_helper(): + return NestedFieldHelper() + + +def test_create_dimension_group_in_nullable_record(jinja_env, nested_field_helper): + """ + REPRODUCTION TEST CASE: + When a field is inside a nullable record (e.g., 'product.created'), + the generated SQL should be '${TABLE}.product.created'. + """ + dimension_group_generator = DimensionGroupGenerator( + timeframes=["time", "date", "week", "month"], + time_suffixes=["_ts"], + primary_key_prefixes=["pk_"], + jinja_env=jinja_env, + nested_field_helper=nested_field_helper, + ) + + # Simulate a field 'created' inside a 'product' record + field = Mock(spec=Field) + field.name = "created_ts" + field.internal_type = "TIMESTAMP" + field.mode = "NULLABLE" + field.description = "Product creation timestamp" + field.parent_field_name = "product" + field.parent_field_type = "RECORD_NULLABLE" + + dimension_group = dimension_group_generator.create_dimension_group(field) + + # EXPECTED: ${TABLE}.product.created_ts + # ACTUAL (Current Bug): ${TABLE}.created_ts + assert dimension_group.sql == "${TABLE}.product.created_ts", ( + f"Expected ${{TABLE}}.product.created_ts but got {dimension_group.sql}" + ) diff --git a/src/tests/test_unnested_view_nested_fields.py b/src/tests/test_unnested_view_nested_fields.py new file mode 100644 index 0000000..4a8d151 --- /dev/null +++ b/src/tests/test_unnested_view_nested_fields.py @@ -0,0 +1,82 @@ +from unittest.mock import Mock, patch +from dwh2looker.lookml_generator.lookml_generator import LookMLGenerator +from dwh2looker.db_client.db_client import Field, Table + + +def test_nested_fields_in_unnested_view(): + """ + REPRODUCTION TEST CASE for deeply nested fields in unnested views. + In an unnested view (e.g., order_items), the SQL for a nested field + (e.g., item_meta.id) should be relative to ${TABLE} (the array element). + """ + # 1. Define fields + order_items_field = Field( + name="order_items", + internal_type="RECORD", + mode="REPEATED", + description="List of items", + ) + item_meta_field = Field( + name="item_meta", + internal_type="RECORD", + mode="NULLABLE", + description="Item metadata", + parent_field_name="order_items", + parent_field_type="RECORD_REPEATED", + ) + meta_trigger_id_field = Field( + name="meta_trigger_id", + internal_type="STRING", + mode="NULLABLE", + description="Trigger ID", + parent_field_name="order_items.item_meta", + parent_field_type="RECORD_NULLABLE", + ) + + # 2. Mock Table and Schema + mock_table = Table(name="fact__interactions", internal_schema=[]) + mock_table.schema = { + order_items_field: {item_meta_field: {meta_trigger_id_field: {}}} + } + + # 3. Setup LookMLGenerator + with patch("dwh2looker.lookml_generator.lookml_generator.Config") as MockConfig: + mock_config = Mock() + MockConfig.return_value = mock_config + mock_config.get_property.return_value = [] + + # Special case for timeframes and prefixes + def get_prop(prop, default=None): + if prop == "primary_key_prefixes": + return ["pk_"] + if prop == "timeframes": + return ["time", "date"] + return default + + mock_config.get_property.side_effect = get_prop + generator = LookMLGenerator(db_type="bigquery") + + # 4. Process views + # We expect two views: fact__interactions and fact__interactions__order_items + views = list( + generator.process_views( + schema=mock_table.schema, + view_name=mock_table.name, + sql_table_name="project.dataset.fact__interactions", + ) + ) + + # 5. Verify the order_items view + # views is a list of tuples: (rendered_string, view_object) + order_items_view_tuple = next( + (v for v in views if v[1].name == "order_items"), None + ) + assert order_items_view_tuple is not None, "order_items view not generated" + + rendered_content, _ = order_items_view_tuple + + # FIND meta_trigger_id dimension + # EXPECTED SQL: ${TABLE}.item_meta.meta_trigger_id + assert "sql: ${TABLE}.item_meta.meta_trigger_id" in rendered_content, ( + f"Incorrect SQL for meta_trigger_id in unnested view. Content:\n{rendered_content}" + )