Skip to content

Commit b66b6ea

Browse files
harshsoni2024harshsoni2024
authored
Fix - 27308: Mark Charts deleted (#27309)
1 parent 677b35e commit b66b6ea

34 files changed

Lines changed: 391 additions & 185 deletions

File tree

‎ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,11 @@ class DashboardServiceTopology(ServiceTopology):
123123
),
124124
],
125125
children=["bulk_data_model", "dashboard"],
126-
post_process=["mark_dashboards_as_deleted", "mark_datamodels_as_deleted"],
126+
post_process=[
127+
"mark_dashboards_as_deleted",
128+
"mark_datamodels_as_deleted",
129+
"mark_charts_as_deleted",
130+
],
127131
)
128132
# Dashboard Services have very different approaches when
129133
# when dealing with data models. Tableau has the models
@@ -218,6 +222,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
218222
context = TopologyContextManager(topology)
219223
dashboard_source_state: Set = set()
220224
datamodel_source_state: Set = set()
225+
chart_source_state: Set = set()
221226

222227
@retry_with_docker_host()
223228
def __init__(
@@ -483,6 +488,20 @@ def mark_datamodels_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
483488
params={"service": self.context.get().dashboard_service},
484489
)
485490

491+
def mark_charts_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
492+
"""
493+
Method to mark the charts as deleted
494+
"""
495+
if self.source_config.markDeletedCharts:
496+
logger.info("Mark Deleted Charts set to True")
497+
yield from delete_entity_from_source(
498+
metadata=self.metadata,
499+
entity_type=Chart,
500+
entity_source_state=self.chart_source_state,
501+
mark_deleted_entity=self.source_config.markDeletedCharts,
502+
params={"service": self.context.get().dashboard_service},
503+
)
504+
486505
def get_owner_ref( # pylint: disable=unused-argument, useless-return
487506
self, dashboard_details
488507
) -> Optional[EntityReferenceList]:
@@ -522,6 +541,19 @@ def register_record_datamodel(
522541

523542
self.datamodel_source_state.add(datamodel_fqn)
524543

544+
def register_record_chart(self, chart_request: CreateChartRequest) -> None:
545+
"""
546+
Mark the chart record as scanned and update the chart_source_state
547+
"""
548+
chart_fqn = fqn.build(
549+
self.metadata,
550+
entity_type=Chart,
551+
service_name=chart_request.service.root,
552+
chart_name=chart_request.name.root,
553+
)
554+
555+
self.chart_source_state.add(chart_fqn)
556+
525557
@staticmethod
526558
def _get_add_lineage_request(
527559
to_entity: Union[Dashboard, DashboardDataModel, Chart],

‎ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -237,20 +237,18 @@ def yield_dashboard_chart(
237237
self.status.filter(chart.name, "Chart Pattern not allowed")
238238
continue
239239
if chart.name:
240-
yield Either(
241-
right=CreateChartRequest(
242-
name=EntityName(str(chart_id)),
243-
description=(
244-
Markdown(chart.description)
245-
if chart.description
246-
else None
247-
),
248-
displayName=chart.name,
249-
sourceUrl=SourceUrl(chart_url),
250-
service=self.context.get().dashboard_service,
251-
chartType=get_standard_chart_type(chart.metadata.chartType),
252-
)
240+
chart_request = CreateChartRequest(
241+
name=EntityName(str(chart_id)),
242+
description=(
243+
Markdown(chart.description) if chart.description else None
244+
),
245+
displayName=chart.name,
246+
sourceUrl=SourceUrl(chart_url),
247+
service=self.context.get().dashboard_service,
248+
chartType=get_standard_chart_type(chart.metadata.chartType),
253249
)
250+
yield Either(right=chart_request)
251+
self.register_record_chart(chart_request=chart_request)
254252
except Exception as exc:
255253
name = chart.name if chart else ""
256254
yield Either(

‎ingestion/src/metadata/ingestion/source/dashboard/grafana/metadata.py‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -224,23 +224,23 @@ def yield_dashboard_chart(
224224
# Map Grafana panel types to standard chart types
225225
chart_type = self._map_panel_type_to_chart_type(panel.type)
226226

227-
yield Either(
228-
right=CreateChartRequest(
229-
name=EntityName(chart_name),
230-
displayName=chart_display_name,
231-
description=(
232-
Markdown(panel.description) if panel.description else None
233-
),
234-
chartType=chart_type,
235-
service=FullyQualifiedEntityName(
236-
self.context.get().dashboard_service
237-
),
238-
sourceUrl=SourceUrl(
239-
f"{clean_uri(self.service_connection.hostPort)}"
240-
f"{dashboard_details.meta.url}?viewPanel={panel.id}"
241-
),
242-
)
227+
chart_request = CreateChartRequest(
228+
name=EntityName(chart_name),
229+
displayName=chart_display_name,
230+
description=(
231+
Markdown(panel.description) if panel.description else None
232+
),
233+
chartType=chart_type,
234+
service=FullyQualifiedEntityName(
235+
self.context.get().dashboard_service
236+
),
237+
sourceUrl=SourceUrl(
238+
f"{clean_uri(self.service_connection.hostPort)}"
239+
f"{dashboard_details.meta.url}?viewPanel={panel.id}"
240+
),
243241
)
242+
yield Either(right=chart_request)
243+
self.register_record_chart(chart_request=chart_request)
244244
except Exception as exc:
245245
yield Either(
246246
left=StackTraceError(

‎ingestion/src/metadata/ingestion/source/dashboard/lightdash/metadata.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -176,18 +176,18 @@ def yield_dashboard_chart(
176176
if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
177177
self.status.filter(chart.name, "Chart Pattern not allowed")
178178
continue
179-
yield Either(
180-
right=CreateChartRequest(
181-
name=EntityName(chart.uuid),
182-
displayName=chart.name,
183-
description=(
184-
Markdown(chart.description) if chart.description else None
185-
),
186-
sourceUrl=SourceUrl(chart_url),
187-
service=self.context.get().dashboard_service,
188-
chartType=chart_type,
189-
)
179+
chart_request = CreateChartRequest(
180+
name=EntityName(chart.uuid),
181+
displayName=chart.name,
182+
description=(
183+
Markdown(chart.description) if chart.description else None
184+
),
185+
sourceUrl=SourceUrl(chart_url),
186+
service=self.context.get().dashboard_service,
187+
chartType=chart_type,
190188
)
189+
yield Either(right=chart_request)
190+
self.register_record_chart(chart_request=chart_request)
191191
self.status.scanned(chart.name)
192192
except Exception as exc: # pylint: disable=broad-except
193193
logger.debug(traceback.format_exc())

‎ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1777,16 +1777,16 @@ def yield_dashboard_chart(
17771777
source_url = chart.result_maker.query.share_url
17781778
else:
17791779
source_url = f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}"
1780-
yield Either(
1781-
right=CreateChartRequest(
1782-
name=EntityName(chart.id),
1783-
displayName=chart.title or chart.id,
1784-
description=Markdown(description) if description else None,
1785-
chartType=get_standard_chart_type(chart.type).value,
1786-
sourceUrl=SourceUrl(source_url),
1787-
service=self.context.get().dashboard_service,
1788-
)
1780+
chart_request = CreateChartRequest(
1781+
name=EntityName(chart.id),
1782+
displayName=chart.title or chart.id,
1783+
description=Markdown(description) if description else None,
1784+
chartType=get_standard_chart_type(chart.type).value,
1785+
sourceUrl=SourceUrl(source_url),
1786+
service=self.context.get().dashboard_service,
17891787
)
1788+
yield Either(right=chart_request)
1789+
self.register_record_chart(chart_request=chart_request)
17901790

17911791
except Exception as exc:
17921792
yield Either(

‎ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -271,16 +271,16 @@ def yield_dashboard_chart(
271271
):
272272
self.status.filter(chart_details.name, "Chart Pattern not allowed")
273273
continue
274-
yield Either(
275-
right=CreateChartRequest(
276-
name=EntityName(chart_details.id),
277-
displayName=chart_details.name,
278-
description=chart_details.description,
279-
chartType=get_standard_chart_type(chart_details.display).value,
280-
sourceUrl=SourceUrl(chart_url),
281-
service=self.context.get().dashboard_service,
282-
)
274+
chart_request = CreateChartRequest(
275+
name=EntityName(chart_details.id),
276+
displayName=chart_details.name,
277+
description=chart_details.description,
278+
chartType=get_standard_chart_type(chart_details.display).value,
279+
sourceUrl=SourceUrl(chart_url),
280+
service=self.context.get().dashboard_service,
283281
)
282+
yield Either(right=chart_request)
283+
self.register_record_chart(chart_request=chart_request)
284284
except KeyError as exc:
285285
yield Either(
286286
left=StackTraceError(

‎ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -323,16 +323,14 @@ def _yield_chart_from_visualization(
323323
self.status.filter(chart.name, "Chart Pattern not allowed")
324324
continue
325325

326-
yield Either(
327-
right=CreateChartRequest(
328-
name=f"{page.key}{chart.key}",
329-
displayName=chart.name,
330-
chartType=get_standard_chart_type(
331-
chart.visualizationType
332-
).value,
333-
service=self.context.get().dashboard_service,
334-
)
326+
chart_request = CreateChartRequest(
327+
name=f"{page.key}{chart.key}",
328+
displayName=chart.name,
329+
chartType=get_standard_chart_type(chart.visualizationType).value,
330+
service=self.context.get().dashboard_service,
335331
)
332+
yield Either(right=chart_request)
333+
self.register_record_chart(chart_request=chart_request)
336334
except Exception as exc:
337335
yield Either(
338336
left=StackTraceError(

‎ingestion/src/metadata/ingestion/source/dashboard/mode/metadata.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -269,15 +269,15 @@ def yield_dashboard_chart(
269269
chart_url = (
270270
f"{clean_uri(self.service_connection.hostPort)}{chart_path}"
271271
)
272-
yield Either(
273-
right=CreateChartRequest(
274-
name=EntityName(chart.get(client.TOKEN)),
275-
displayName=chart_name,
276-
chartType=ChartType.Other,
277-
sourceUrl=SourceUrl(chart_url),
278-
service=self.context.get().dashboard_service,
279-
)
272+
chart_request = CreateChartRequest(
273+
name=EntityName(chart.get(client.TOKEN)),
274+
displayName=chart_name,
275+
chartType=ChartType.Other,
276+
sourceUrl=SourceUrl(chart_url),
277+
service=self.context.get().dashboard_service,
280278
)
279+
yield Either(right=chart_request)
280+
self.register_record_chart(chart_request=chart_request)
281281
except Exception as exc:
282282
name = chart_name if chart_name else ""
283283
yield Either(

‎ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -536,23 +536,23 @@ def yield_dashboard_chart(
536536
)
537537
continue
538538
self.dashboard_charts[dashboard_details.id].append(chart.id)
539-
yield Either(
540-
right=CreateChartRequest(
541-
name=EntityName(chart.id),
542-
displayName=chart_display_name,
543-
chartType=ChartType.Other.value,
544-
sourceUrl=SourceUrl(
545-
self._get_chart_url(
546-
report_id=chart.reportId,
547-
workspace_id=self.context.get().workspace.id,
548-
dashboard_id=dashboard_details.id,
549-
)
550-
),
551-
service=FullyQualifiedEntityName(
552-
self.context.get().dashboard_service
553-
),
554-
)
539+
chart_request = CreateChartRequest(
540+
name=EntityName(chart.id),
541+
displayName=chart_display_name,
542+
chartType=ChartType.Other.value,
543+
sourceUrl=SourceUrl(
544+
self._get_chart_url(
545+
report_id=chart.reportId,
546+
workspace_id=self.context.get().workspace.id,
547+
dashboard_id=dashboard_details.id,
548+
)
549+
),
550+
service=FullyQualifiedEntityName(
551+
self.context.get().dashboard_service
552+
),
555553
)
554+
yield Either(right=chart_request)
555+
self.register_record_chart(chart_request=chart_request)
556556
except Exception as exc:
557557
yield Either(
558558
left=StackTraceError(

‎ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -341,20 +341,20 @@ def yield_dashboard_chart(
341341
):
342342
self.status.filter(chart.qMeta.title, "Chart Pattern not allowed")
343343
continue
344-
yield Either(
345-
right=CreateChartRequest(
346-
name=EntityName(chart.qInfo.qId),
347-
displayName=chart.qMeta.title,
348-
description=(
349-
Markdown(chart.qMeta.description)
350-
if chart.qMeta.description
351-
else None
352-
),
353-
chartType=ChartType.Other,
354-
sourceUrl=SourceUrl(chart_url),
355-
service=self.context.get().dashboard_service,
356-
)
344+
chart_request = CreateChartRequest(
345+
name=EntityName(chart.qInfo.qId),
346+
displayName=chart.qMeta.title,
347+
description=(
348+
Markdown(chart.qMeta.description)
349+
if chart.qMeta.description
350+
else None
351+
),
352+
chartType=ChartType.Other,
353+
sourceUrl=SourceUrl(chart_url),
354+
service=self.context.get().dashboard_service,
357355
)
356+
yield Either(right=chart_request)
357+
self.register_record_chart(chart_request=chart_request)
358358
except Exception as exc: # pylint: disable=broad-except
359359
yield Either(
360360
left=StackTraceError(

0 commit comments

Comments
 (0)