Skip to content

Fix - 27308: Mark Charts deleted #27309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
harshsoni2024 merged 8 commits into from
Apr 17, 2026
Merged

Fix - 27308: Mark Charts deleted #27309

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
49f7aa2
mark charts as deleted when deleted from source
harshsoni2024 Apr 13, 2026
cd4e55b
Merge branch 'main' into mark_charts_deleted
harshsoni2024 Apr 13, 2026
6bd7c2b
Update generated TypeScript types
github-actions[bot] Apr 13, 2026
1e19018
Merge branch 'main' into mark_charts_deleted
harshsoni2024 Apr 14, 2026
b5a82ef
remove redundant code
harshsoni2024 Apr 14, 2026
134c339
tests add
harshsoni2024 Apr 14, 2026
575736e
quicksight fix
harshsoni2024 Apr 14, 2026
38b8f1a
Merge branch 'main' into mark_charts_deleted
harshsoni2024 Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,11 @@ class DashboardServiceTopology(ServiceTopology):
),
],
children=["bulk_data_model", "dashboard"],
post_process=["mark_dashboards_as_deleted", "mark_datamodels_as_deleted"],
post_process=[
"mark_dashboards_as_deleted",
"mark_datamodels_as_deleted",
"mark_charts_as_deleted",
],
)
# Dashboard Services have very different approaches when
# when dealing with data models. Tableau has the models
Expand Down Expand Up @@ -218,6 +222,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
context = TopologyContextManager(topology)
dashboard_source_state: Set = set()
datamodel_source_state: Set = set()
chart_source_state: Set = set()

@retry_with_docker_host()
def __init__(
Expand Down Expand Up @@ -483,6 +488,20 @@ def mark_datamodels_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
params={"service": self.context.get().dashboard_service},
)

def mark_charts_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
"""
Method to mark the charts as deleted
"""
if self.source_config.markDeletedCharts:
Comment thread
harshsoni2024 marked this conversation as resolved.
logger.info("Mark Deleted Charts set to True")
yield from delete_entity_from_source(
metadata=self.metadata,
entity_type=Chart,
entity_source_state=self.chart_source_state,
mark_deleted_entity=self.source_config.markDeletedCharts,
params={"service": self.context.get().dashboard_service},
)
Comment thread
harshsoni2024 marked this conversation as resolved.

def get_owner_ref( # pylint: disable=unused-argument, useless-return
self, dashboard_details
) -> Optional[EntityReferenceList]:
Expand Down Expand Up @@ -522,6 +541,19 @@ def register_record_datamodel(

self.datamodel_source_state.add(datamodel_fqn)

def register_record_chart(self, chart_request: CreateChartRequest) -> None:
"""
Mark the chart record as scanned and update the chart_source_state
"""
chart_fqn = fqn.build(
self.metadata,
entity_type=Chart,
service_name=chart_request.service.root,
chart_name=chart_request.name.root,
)

self.chart_source_state.add(chart_fqn)

@staticmethod
def _get_add_lineage_request(
to_entity: Union[Dashboard, DashboardDataModel, Chart],
Expand Down
24 changes: 11 additions & 13 deletions ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,20 +237,18 @@ def yield_dashboard_chart(
self.status.filter(chart.name, "Chart Pattern not allowed")
continue
if chart.name:
yield Either(
right=CreateChartRequest(
name=EntityName(str(chart_id)),
description=(
Markdown(chart.description)
if chart.description
else None
),
displayName=chart.name,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
chartType=get_standard_chart_type(chart.metadata.chartType),
)
chart_request = CreateChartRequest(
name=EntityName(str(chart_id)),
description=(
Markdown(chart.description) if chart.description else None
),
displayName=chart.name,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
chartType=get_standard_chart_type(chart.metadata.chartType),
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except Exception as exc:
name = chart.name if chart else ""
yield Either(
Expand Down
32 changes: 16 additions & 16 deletions ingestion/src/metadata/ingestion/source/dashboard/grafana/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,23 +224,23 @@ def yield_dashboard_chart(
# Map Grafana panel types to standard chart types
chart_type = self._map_panel_type_to_chart_type(panel.type)

yield Either(
right=CreateChartRequest(
name=EntityName(chart_name),
displayName=chart_display_name,
description=(
Markdown(panel.description) if panel.description else None
),
chartType=chart_type,
service=FullyQualifiedEntityName(
self.context.get().dashboard_service
),
sourceUrl=SourceUrl(
f"{clean_uri(self.service_connection.hostPort)}"
f"{dashboard_details.meta.url}?viewPanel={panel.id}"
),
)
chart_request = CreateChartRequest(
name=EntityName(chart_name),
displayName=chart_display_name,
description=(
Markdown(panel.description) if panel.description else None
),
chartType=chart_type,
service=FullyQualifiedEntityName(
self.context.get().dashboard_service
),
sourceUrl=SourceUrl(
f"{clean_uri(self.service_connection.hostPort)}"
f"{dashboard_details.meta.url}?viewPanel={panel.id}"
),
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
Copy link
Copy Markdown
Member

@pmbrull pmbrull Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we take the chance to improve the framework a bit. I think we have to keep adding this register_record_xyz when we miss it for this or that entity. Maybe we should rely on the yield Either right and the topology itself to say "entity registered" so we know we are not missing any connectors at all. thoughts?

Copy link
Copy Markdown
Contributor Author

@harshsoni2024 harshsoni2024 Apr 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this improvement makes sense. since we have to manually keep that in each connector X for each entity(for e.g. 3 in case of dashboard).
We can couple this with topology manager to keep source state to maintain unchaged/changed/deleted entities & update them on ingestion based on markDeleted flag.
I am thinking of handling this in different pr something like this, wdyt ?
#27450

except Exception as exc:
yield Either(
left=StackTraceError(
Expand Down
22 changes: 11 additions & 11 deletions ingestion/src/metadata/ingestion/source/dashboard/lightdash/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,18 +176,18 @@ def yield_dashboard_chart(
if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
self.status.filter(chart.name, "Chart Pattern not allowed")
continue
yield Either(
right=CreateChartRequest(
name=EntityName(chart.uuid),
displayName=chart.name,
description=(
Markdown(chart.description) if chart.description else None
),
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
chartType=chart_type,
)
chart_request = CreateChartRequest(
name=EntityName(chart.uuid),
displayName=chart.name,
description=(
Markdown(chart.description) if chart.description else None
),
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
chartType=chart_type,
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
self.status.scanned(chart.name)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
Expand Down
18 changes: 9 additions & 9 deletions ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1777,16 +1777,16 @@ def yield_dashboard_chart(
source_url = chart.result_maker.query.share_url
else:
source_url = f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}"
yield Either(
right=CreateChartRequest(
name=EntityName(chart.id),
displayName=chart.title or chart.id,
description=Markdown(description) if description else None,
chartType=get_standard_chart_type(chart.type).value,
sourceUrl=SourceUrl(source_url),
service=self.context.get().dashboard_service,
)
chart_request = CreateChartRequest(
name=EntityName(chart.id),
displayName=chart.title or chart.id,
description=Markdown(description) if description else None,
chartType=get_standard_chart_type(chart.type).value,
sourceUrl=SourceUrl(source_url),
service=self.context.get().dashboard_service,
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)

except Exception as exc:
yield Either(
Expand Down
18 changes: 9 additions & 9 deletions ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,16 +271,16 @@ def yield_dashboard_chart(
):
self.status.filter(chart_details.name, "Chart Pattern not allowed")
continue
yield Either(
right=CreateChartRequest(
name=EntityName(chart_details.id),
displayName=chart_details.name,
description=chart_details.description,
chartType=get_standard_chart_type(chart_details.display).value,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
)
chart_request = CreateChartRequest(
name=EntityName(chart_details.id),
displayName=chart_details.name,
description=chart_details.description,
chartType=get_standard_chart_type(chart_details.display).value,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except KeyError as exc:
yield Either(
left=StackTraceError(
Expand Down
16 changes: 7 additions & 9 deletions ingestion/src/metadata/ingestion/source/dashboard/microstrategy/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,16 +323,14 @@ def _yield_chart_from_visualization(
self.status.filter(chart.name, "Chart Pattern not allowed")
continue

yield Either(
right=CreateChartRequest(
name=f"{page.key}{chart.key}",
displayName=chart.name,
chartType=get_standard_chart_type(
chart.visualizationType
).value,
service=self.context.get().dashboard_service,
)
chart_request = CreateChartRequest(
name=f"{page.key}{chart.key}",
displayName=chart.name,
chartType=get_standard_chart_type(chart.visualizationType).value,
service=self.context.get().dashboard_service,
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except Exception as exc:
yield Either(
left=StackTraceError(
Expand Down
16 changes: 8 additions & 8 deletions ingestion/src/metadata/ingestion/source/dashboard/mode/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,15 +269,15 @@ def yield_dashboard_chart(
chart_url = (
f"{clean_uri(self.service_connection.hostPort)}{chart_path}"
)
yield Either(
right=CreateChartRequest(
name=EntityName(chart.get(client.TOKEN)),
displayName=chart_name,
chartType=ChartType.Other,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
)
chart_request = CreateChartRequest(
name=EntityName(chart.get(client.TOKEN)),
displayName=chart_name,
chartType=ChartType.Other,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except Exception as exc:
name = chart_name if chart_name else ""
yield Either(
Expand Down
34 changes: 18 additions & 16 deletions ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,8 @@
if isinstance(dashboard_details, PowerBIDashboard):
self.dashboard_charts[dashboard_details.id] = []
charts = dashboard_details.tiles
if dashboard_details.displayName == "Human Resources Sample":
pass

Check warning on line 528 in ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py

View check run for this annotation

SonarQubeCloud / [open-metadata-ingestion] SonarCloud Code Analysis

Either remove or fill this block of code. 


See more on https://sonarcloud.io/project/issues?id=open-metadata-ingestion&issues=AZ2F08F-srmu-Gaxxy4z&open=AZ2F08F-srmu-Gaxxy4z&pullRequest=27309
Comment thread
harshsoni2024 marked this conversation as resolved.
Outdated
for chart in charts or []:
try:
chart_title = chart.title
Expand All @@ -536,23 +538,23 @@
)
continue
self.dashboard_charts[dashboard_details.id].append(chart.id)
yield Either(
right=CreateChartRequest(
name=EntityName(chart.id),
displayName=chart_display_name,
chartType=ChartType.Other.value,
sourceUrl=SourceUrl(
self._get_chart_url(
report_id=chart.reportId,
workspace_id=self.context.get().workspace.id,
dashboard_id=dashboard_details.id,
)
),
service=FullyQualifiedEntityName(
self.context.get().dashboard_service
),
)
chart_request = CreateChartRequest(
name=EntityName(chart.id),
displayName=chart_display_name,
chartType=ChartType.Other.value,
sourceUrl=SourceUrl(
self._get_chart_url(
report_id=chart.reportId,
workspace_id=self.context.get().workspace.id,
dashboard_id=dashboard_details.id,
)
),
service=FullyQualifiedEntityName(
self.context.get().dashboard_service
),
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except Exception as exc:
yield Either(
left=StackTraceError(
Expand Down
26 changes: 13 additions & 13 deletions ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,20 +341,20 @@ def yield_dashboard_chart(
):
self.status.filter(chart.qMeta.title, "Chart Pattern not allowed")
continue
yield Either(
right=CreateChartRequest(
name=EntityName(chart.qInfo.qId),
displayName=chart.qMeta.title,
description=(
Markdown(chart.qMeta.description)
if chart.qMeta.description
else None
),
chartType=ChartType.Other,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
)
chart_request = CreateChartRequest(
name=EntityName(chart.qInfo.qId),
displayName=chart.qMeta.title,
description=(
Markdown(chart.qMeta.description)
if chart.qMeta.description
else None
),
chartType=ChartType.Other,
sourceUrl=SourceUrl(chart_url),
service=self.context.get().dashboard_service,
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except Exception as exc: # pylint: disable=broad-except
yield Either(
left=StackTraceError(
Expand Down
30 changes: 15 additions & 15 deletions ingestion/src/metadata/ingestion/source/dashboard/qliksense/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,22 +192,22 @@ def yield_dashboard_chart(
):
self.status.filter(chart.qMeta.title, "Chart Pattern not allowed")
continue
yield Either(
right=CreateChartRequest(
name=EntityName(chart.qInfo.qId),
displayName=chart.qMeta.title,
description=(
Markdown(chart.qMeta.description)
if chart.qMeta.description
else None
),
chartType=ChartType.Other,
sourceUrl=SourceUrl(chart_url),
service=FullyQualifiedEntityName(
self.context.get().dashboard_service
),
)
chart_request = CreateChartRequest(
name=EntityName(chart.qInfo.qId),
displayName=chart.qMeta.title,
description=(
Markdown(chart.qMeta.description)
if chart.qMeta.description
else None
),
chartType=ChartType.Other,
sourceUrl=SourceUrl(chart_url),
service=FullyQualifiedEntityName(
self.context.get().dashboard_service
),
)
yield Either(right=chart_request)
self.register_record_chart(chart_request=chart_request)
except Exception as exc: # pylint: disable=broad-except
yield Either(
left=StackTraceError(
Expand Down
Loading
Loading