From fa53a8bd7973d3061be1b8de1b007a66e01e005a Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 05:30:55 -0500 Subject: [PATCH 1/2] feat: add databricks delta monitoring dashboard --- Cargo.lock | 8 +- README.md | 21 + docs/monitoring.md | 51 ++ justfile | 6 + platform/databricks/delta/README.md | 4 + .../databricks/delta/dashboards/README.md | 19 + .../convex_sync_overview.lvdash.json.tmpl | 536 ++++++++++++++++++ .../delta/dashboards/dashboards.json | 10 + scripts/publish-databricks-delta-dashboard.sh | 60 ++ scripts/render-databricks-delta-dashboard.sh | 46 ++ 10 files changed, 757 insertions(+), 4 deletions(-) create mode 100644 docs/monitoring.md create mode 100644 platform/databricks/delta/dashboards/README.md create mode 100644 platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl create mode 100644 platform/databricks/delta/dashboards/dashboards.json create mode 100755 scripts/publish-databricks-delta-dashboard.sh create mode 100755 scripts/render-databricks-delta-dashboard.sh diff --git a/Cargo.lock b/Cargo.lock index 08ba30c..7d82ab3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -867,7 +867,7 @@ dependencies = [ [[package]] name = "convex-export-s3" -version = "0.0.1" +version = "0.0.3" dependencies = [ "arrow-array", "arrow-schema", @@ -887,7 +887,7 @@ dependencies = [ [[package]] name = "convex-inspect" -version = "0.0.2" +version = "0.0.3" dependencies = [ "clap", "convex-sync-core", @@ -898,7 +898,7 @@ dependencies = [ [[package]] name = "convex-sync" -version = "0.0.2" +version = "0.0.3" dependencies = [ "clap", "convex-export-s3", @@ -910,7 +910,7 @@ dependencies = [ [[package]] name = "convex-sync-core" -version = "0.0.1" +version = "0.0.3" dependencies = [ "clap", "hex", diff --git a/README.md b/README.md index 61311f1..a3da78e 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,26 @@ depot ci run --workflow .depot/workflows/release.yml depot ci run --workflow .depot/workflows/release-rc.yml ``` +## Monitoring + +The Delta path now has a Lakeview dashboard template and publish flow for a +high-level sync view: + +```bash +just databricks-delta-publish-dashboard DEFAULT +``` + +The first version focuses on: + +- latest checkpoint freshness +- bronze table count +- silver table count +- recent checkpoint history +- bronze and silver table inventory + +Silver remaining empty is expected until you deploy a real Lakeflow `AUTO CDC` +pipeline that reads bronze and materializes silver. + ## Suggested Screenshots If you want to show this repo working in a talk or video, start with: @@ -183,6 +203,7 @@ There is a more detailed capture list in [docs/demo-storyboard.md](docs/demo-sto - [Ask DeepWiki about this repo](https://deepwiki.com/shpitdev/convex-sync-kit) - [docs/architecture.md](docs/architecture.md) +- [docs/monitoring.md](docs/monitoring.md) - [docs/public-reference-map.md](docs/public-reference-map.md) - [docs/release-artifacts.md](docs/release-artifacts.md) - [docs/demo-storyboard.md](docs/demo-storyboard.md) diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..b10aa32 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,51 @@ +# Monitoring + +## Databricks Delta Dashboard + +This repo now includes a Lakeview dashboard template for the Databricks Delta +path: + +- `platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl` +- `platform/databricks/delta/dashboards/dashboards.json` + +Publish it with: + +```bash +just databricks-delta-publish-dashboard DEFAULT +``` + +If you pass an existing dashboard ID to the publish script, it updates and +re-publishes that dashboard. Otherwise it creates a new one and prints the new +dashboard ID. + +The first dashboard focuses on: + +- latest checkpoint freshness +- bronze table count +- silver table count +- recent checkpoint history +- bronze and silver table inventory + +## Why Silver Is Empty + +Silver is still expected to be empty until a real Lakeflow `AUTO CDC` pipeline +is deployed. + +What exists today: + +- control schema and checkpoint tables +- bronze CDC landing +- Delta extractor job +- Lakeflow SQL template for per-table `AUTO CDC` + +What is still missing: + +- a source-aware pipeline generation and deploy path that turns the bronze + tables for a specific source into a real deployed Lakeflow pipeline + +The generic blocker is not Lakeflow itself. It is the missing code path that: + +1. enumerates the bronze tables for one source +2. decides the silver target names consistently +3. renders the per-table `AUTO CDC` SQL +4. deploys or updates the pipeline repeatably diff --git a/justfile b/justfile index bea2c9a..7ccf104 100644 --- a/justfile +++ b/justfile @@ -81,6 +81,12 @@ databricks-delta-sync-secret *args: databricks-delta-bootstrap warehouse_id profile="DEFAULT": ./scripts/bootstrap-databricks-delta.sh {{profile}} {{warehouse_id}} +databricks-delta-render-dashboard output_file: + ./scripts/render-databricks-delta-dashboard.sh {{output_file}} + +databricks-delta-publish-dashboard profile warehouse_id dashboard_id="": + ./scripts/publish-databricks-delta-dashboard.sh {{profile}} {{warehouse_id}} {{dashboard_id}} + databricks-delta-deploy profile="DEFAULT" target="dev": ./scripts/deploy-databricks-delta.sh {{profile}} {{target}} diff --git a/platform/databricks/delta/README.md b/platform/databricks/delta/README.md index ed63719..c309de8 100644 --- a/platform/databricks/delta/README.md +++ b/platform/databricks/delta/README.md @@ -29,6 +29,7 @@ cannot overwrite key, ordering, or delete semantics. - `databricks.yml`: Databricks bundle entrypoint - `resources/`: Databricks bundle resources for the extractor job +- `dashboards/`: Lakeview dashboard templates and metadata - `extractor/convex_cdc_job.py`: Databricks job entrypoint - `sql/bootstrap/`: ordered bootstrap DDL for configurable control/bronze/silver schemas and checkpoint table - `lakeflow/bronze_to_silver_template.sql`: per-table Lakeflow template @@ -39,6 +40,8 @@ Bundle lifecycle: - `scripts/ensure-databricks-delta-secret.sh [scope] [key]` - `scripts/bootstrap-databricks-delta.sh ` +- `scripts/render-databricks-delta-dashboard.sh ` +- `scripts/publish-databricks-delta-dashboard.sh [dashboard_id]` - `scripts/deploy-databricks-delta.sh ` - `scripts/run-databricks-delta-job.sh [job_key]` - `scripts/run-databricks-delta-smoke.sh ` @@ -105,6 +108,7 @@ Recommended operator entrypoints: ```bash just databricks-delta-sync-secret just databricks-delta-bootstrap +just databricks-delta-publish-dashboard DEFAULT just databricks-delta-deploy just databricks-delta-run just databricks-delta-smoke diff --git a/platform/databricks/delta/dashboards/README.md b/platform/databricks/delta/dashboards/README.md new file mode 100644 index 0000000..39fc40b --- /dev/null +++ b/platform/databricks/delta/dashboards/README.md @@ -0,0 +1,19 @@ +# Databricks Delta Dashboards + +Lakeview dashboard assets for monitoring the Convex sync at a high level. + +Files: + +- `dashboards.json`: lightweight manifest for the included dashboards +- `convex_sync_overview.lvdash.json.tmpl`: renderable Lakeview dashboard template + +Use the helper scripts: + +```bash +./scripts/render-databricks-delta-dashboard.sh /tmp/convex-sync-overview.lvdash.json +./scripts/publish-databricks-delta-dashboard.sh DEFAULT +``` + +If you pass an existing dashboard ID to the publish script, it updates and +re-publishes that dashboard. Otherwise it creates a new one and prints the +dashboard ID. diff --git a/platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl b/platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl new file mode 100644 index 0000000..1b23d86 --- /dev/null +++ b/platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl @@ -0,0 +1,536 @@ +{ + "datasets": [ + { + "name": "sync_summary", + "displayName": "Sync Summary", + "config": { + "source": "WITH checkpoint_latest AS ( SELECT source_id, phase, updated_at FROM `{{CATALOG}}`.`{{CONTROL_SCHEMA}}`.connector_checkpoint_latest ), checkpoint_counts AS ( SELECT source_id, COUNT(*) AS checkpoint_write_count FROM `{{CATALOG}}`.`{{CONTROL_SCHEMA}}`.`{{CHECKPOINT_TABLE}}` GROUP BY source_id ), bronze_tables AS ( SELECT COUNT(*) AS bronze_table_count FROM `{{CATALOG}}`.information_schema.tables WHERE table_schema = '{{BRONZE_SCHEMA}}' ), silver_tables AS ( SELECT COUNT(*) AS silver_table_count FROM `{{CATALOG}}`.information_schema.tables WHERE table_schema = '{{SILVER_SCHEMA}}' ) SELECT l.source_id, l.phase, l.updated_at, ROUND((UNIX_TIMESTAMP(CURRENT_TIMESTAMP()) - UNIX_TIMESTAMP(l.updated_at)) / 60.0, 2) AS checkpoint_lag_minutes, COALESCE(c.checkpoint_write_count, 0) AS checkpoint_write_count, b.bronze_table_count, s.silver_table_count FROM checkpoint_latest l LEFT JOIN checkpoint_counts c ON c.source_id = l.source_id CROSS JOIN bronze_tables b CROSS JOIN silver_tables s", + "measures": [ + { + "name": "Bronze Tables", + "expr": "MAX(bronze_table_count)", + "displayName": "Bronze Tables", + "format": { + "type": "number", + "decimalPlaces": { + "type": "exact", + "places": 0 + } + } + }, + { + "name": "Silver Tables", + "expr": "MAX(silver_table_count)", + "displayName": "Silver Tables", + "format": { + "type": "number", + "decimalPlaces": { + "type": "exact", + "places": 0 + } + } + }, + { + "name": "Checkpoint Lag Minutes", + "expr": "MAX(checkpoint_lag_minutes)", + "displayName": "Checkpoint Lag Minutes", + "format": { + "type": "number", + "decimalPlaces": { + "type": "exact", + "places": 2 + } + } + }, + { + "name": "Checkpoint Writes", + "expr": "MAX(checkpoint_write_count)", + "displayName": "Checkpoint Writes", + "format": { + "type": "number", + "decimalPlaces": { + "type": "exact", + "places": 0 + } + } + } + ], + "dimensions": [ + { + "name": "Source ID", + "expr": "source_id", + "displayName": "Source ID" + }, + { + "name": "Phase", + "expr": "phase", + "displayName": "Phase" + }, + { + "name": "Updated At", + "expr": "updated_at", + "displayName": "Updated At" + } + ], + "version": 1.1, + "comment": "High-level sync summary for {{SOURCE_LABEL}}." + } + }, + { + "name": "recent_checkpoints", + "displayName": "Recent Checkpoints", + "config": { + "source": "SELECT source_id, phase, snapshot_ts, delta_cursor, run_id, updated_at FROM `{{CATALOG}}`.`{{CONTROL_SCHEMA}}`.`{{CHECKPOINT_TABLE}}` ORDER BY updated_at DESC LIMIT 50", + "dimensions": [ + { + "name": "Source ID", + "expr": "source_id", + "displayName": "Source ID" + }, + { + "name": "Phase", + "expr": "phase", + "displayName": "Phase" + }, + { + "name": "Snapshot TS", + "expr": "snapshot_ts", + "displayName": "Snapshot TS" + }, + { + "name": "Delta Cursor", + "expr": "delta_cursor", + "displayName": "Delta Cursor" + }, + { + "name": "Run ID", + "expr": "run_id", + "displayName": "Run ID" + }, + { + "name": "Updated At", + "expr": "updated_at", + "displayName": "Updated At" + } + ], + "version": 1.1, + "comment": "Recent checkpoint writes." + } + }, + { + "name": "delta_table_inventory", + "displayName": "Delta Table Inventory", + "config": { + "source": "SELECT table_schema AS layer, table_name, table_type, created, last_altered FROM `{{CATALOG}}`.information_schema.tables WHERE table_schema IN ('{{BRONZE_SCHEMA}}', '{{SILVER_SCHEMA}}') ORDER BY table_schema, table_name", + "dimensions": [ + { + "name": "Layer", + "expr": "layer", + "displayName": "Layer" + }, + { + "name": "Table Name", + "expr": "table_name", + "displayName": "Table Name" + }, + { + "name": "Table Type", + "expr": "table_type", + "displayName": "Table Type" + }, + { + "name": "Created", + "expr": "created", + "displayName": "Created" + }, + { + "name": "Last Altered", + "expr": "last_altered", + "displayName": "Last Altered" + } + ], + "version": 1.1, + "comment": "Bronze and silver table inventory." + } + } + ], + "pages": [ + { + "name": "convex_sync_overview", + "displayName": "Convex Sync Overview", + "layout": [ + { + "widget": { + "name": "title_box", + "multilineTextboxSpec": { + "lines": [ + "# Convex Sync Overview\n", + "High-level sync health for {{SOURCE_LABEL}}. Silver stays empty until you deploy a Lakeflow AUTO CDC pipeline." + ] + } + }, + "position": { + "x": 0, + "y": 0, + "width": 12, + "height": 2 + } + }, + { + "widget": { + "name": "bronze_table_counter", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "sync_summary", + "fields": [ + { + "name": "measure(Bronze Tables)", + "expression": "MEASURE(`Bronze Tables`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "measure(Bronze Tables)", + "displayName": "Bronze Tables" + } + }, + "frame": { + "showTitle": true, + "title": "Bronze Tables" + } + } + }, + "position": { + "x": 0, + "y": 2, + "width": 3, + "height": 3 + } + }, + { + "widget": { + "name": "silver_table_counter", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "sync_summary", + "fields": [ + { + "name": "measure(Silver Tables)", + "expression": "MEASURE(`Silver Tables`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "measure(Silver Tables)", + "displayName": "Silver Tables" + } + }, + "frame": { + "showTitle": true, + "title": "Silver Tables" + } + } + }, + "position": { + "x": 3, + "y": 2, + "width": 3, + "height": 3 + } + }, + { + "widget": { + "name": "checkpoint_lag_counter", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "sync_summary", + "fields": [ + { + "name": "measure(Checkpoint Lag Minutes)", + "expression": "MEASURE(`Checkpoint Lag Minutes`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "measure(Checkpoint Lag Minutes)", + "displayName": "Checkpoint Lag Minutes" + } + }, + "frame": { + "showTitle": true, + "title": "Checkpoint Lag (min)" + } + } + }, + "position": { + "x": 6, + "y": 2, + "width": 3, + "height": 3 + } + }, + { + "widget": { + "name": "checkpoint_writes_counter", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "sync_summary", + "fields": [ + { + "name": "measure(Checkpoint Writes)", + "expression": "MEASURE(`Checkpoint Writes`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "measure(Checkpoint Writes)", + "displayName": "Checkpoint Writes" + } + }, + "frame": { + "showTitle": true, + "title": "Checkpoint Writes" + } + } + }, + "position": { + "x": 9, + "y": 2, + "width": 3, + "height": 3 + } + }, + { + "widget": { + "name": "latest_status_table", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "sync_summary", + "fields": [ + { + "name": "Source ID", + "expression": "`Source ID`" + }, + { + "name": "Phase", + "expression": "`Phase`" + }, + { + "name": "Updated At", + "expression": "`Updated At`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "table", + "rowsPerPage": 5, + "frame": { + "showTitle": true, + "title": "Latest Checkpoint" + }, + "encodings": { + "columns": [ + { + "fieldName": "Source ID", + "displayName": "Source ID" + }, + { + "fieldName": "Phase", + "displayName": "Phase" + }, + { + "fieldName": "Updated At", + "displayName": "Updated At" + } + ] + } + } + }, + "position": { + "x": 0, + "y": 5, + "width": 5, + "height": 5 + } + }, + { + "widget": { + "name": "recent_checkpoints_table", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "recent_checkpoints", + "fields": [ + { + "name": "Source ID", + "expression": "`Source ID`" + }, + { + "name": "Phase", + "expression": "`Phase`" + }, + { + "name": "Delta Cursor", + "expression": "`Delta Cursor`" + }, + { + "name": "Updated At", + "expression": "`Updated At`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "table", + "rowsPerPage": 10, + "frame": { + "showTitle": true, + "title": "Recent Checkpoints" + }, + "encodings": { + "columns": [ + { + "fieldName": "Source ID", + "displayName": "Source ID" + }, + { + "fieldName": "Phase", + "displayName": "Phase" + }, + { + "fieldName": "Delta Cursor", + "displayName": "Delta Cursor" + }, + { + "fieldName": "Updated At", + "displayName": "Updated At" + } + ] + } + } + }, + "position": { + "x": 5, + "y": 5, + "width": 7, + "height": 5 + } + }, + { + "widget": { + "name": "table_inventory_table", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "delta_table_inventory", + "fields": [ + { + "name": "Layer", + "expression": "`Layer`" + }, + { + "name": "Table Name", + "expression": "`Table Name`" + }, + { + "name": "Table Type", + "expression": "`Table Type`" + }, + { + "name": "Last Altered", + "expression": "`Last Altered`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "table", + "rowsPerPage": 25, + "frame": { + "showTitle": true, + "title": "Bronze And Silver Inventory" + }, + "encodings": { + "columns": [ + { + "fieldName": "Layer", + "displayName": "Layer" + }, + { + "fieldName": "Table Name", + "displayName": "Table Name" + }, + { + "fieldName": "Table Type", + "displayName": "Table Type" + }, + { + "fieldName": "Last Altered", + "displayName": "Last Altered" + } + ] + } + } + }, + "position": { + "x": 0, + "y": 10, + "width": 12, + "height": 10 + } + } + ], + "pageType": "PAGE" + } + ] +} diff --git a/platform/databricks/delta/dashboards/dashboards.json b/platform/databricks/delta/dashboards/dashboards.json new file mode 100644 index 0000000..eda456e --- /dev/null +++ b/platform/databricks/delta/dashboards/dashboards.json @@ -0,0 +1,10 @@ +{ + "dashboards": [ + { + "key": "convex_sync_overview", + "display_name": "Convex Sync Overview", + "template_file": "platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl", + "purpose": "Checkpoint freshness, bronze coverage, and silver readiness for a single Convex source." + } + ] +} diff --git a/scripts/publish-databricks-delta-dashboard.sh b/scripts/publish-databricks-delta-dashboard.sh new file mode 100755 index 0000000..e76b4dc --- /dev/null +++ b/scripts/publish-databricks-delta-dashboard.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -lt 2 || "$#" -gt 3 ]]; then + echo "usage: $0 [dashboard_id]" >&2 + exit 1 +fi + +profile="$1" +warehouse_id="$2" +dashboard_id="${3:-${DATABRICKS_DELTA_DASHBOARD_ID:-}}" + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" + +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-${CONVEX_SYNC_SOURCE:-default}}" +source_label="${CONVEX_SYNC_SOURCE_LABEL:-$source_slug}" +display_name="${DATABRICKS_DELTA_DASHBOARD_NAME:-Convex Sync Overview (${source_label})}" + +render_dir="$(mktemp -d)" +trap 'rm -rf "$render_dir"' EXIT +rendered_dashboard="$render_dir/convex_sync_overview.lvdash.json" + +"$repo_root/scripts/render-databricks-delta-dashboard.sh" "$rendered_dashboard" >/dev/null + +payload="$render_dir/payload.json" +python - "$rendered_dashboard" "$display_name" "$warehouse_id" "$payload" <<'PY' +import json +import pathlib +import sys + +dashboard_path = pathlib.Path(sys.argv[1]) +display_name = sys.argv[2] +warehouse_id = sys.argv[3] +payload_path = pathlib.Path(sys.argv[4]) + +payload = { + "display_name": display_name, + "warehouse_id": warehouse_id, + "serialized_dashboard": dashboard_path.read_text(), +} +payload_path.write_text(json.dumps(payload)) +PY + +if [[ -n "$dashboard_id" ]]; then + databricks lakeview update "$dashboard_id" -p "$profile" --json @"$payload" >/dev/null + databricks lakeview publish "$dashboard_id" -p "$profile" --warehouse-id "$warehouse_id" >/dev/null + echo "updated dashboard_id=$dashboard_id" +else + create_output="$(databricks lakeview create -p "$profile" --json @"$payload" -o json)" + dashboard_id="$(jq -r '.dashboard_id' <<<"$create_output")" + databricks lakeview publish "$dashboard_id" -p "$profile" --warehouse-id "$warehouse_id" >/dev/null + echo "created dashboard_id=$dashboard_id" +fi + +echo "display_name=$display_name" +echo "warehouse_id=$warehouse_id" diff --git a/scripts/render-databricks-delta-dashboard.sh b/scripts/render-databricks-delta-dashboard.sh new file mode 100755 index 0000000..5334d64 --- /dev/null +++ b/scripts/render-databricks-delta-dashboard.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +output_file="$1" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" + +catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-${CONVEX_SYNC_SOURCE:-default}}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" +checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" +source_label="${CONVEX_SYNC_SOURCE_LABEL:-$source_slug}" + +python - "$repo_root" "$output_file" "$catalog" "$control_schema" "$bronze_schema" "$silver_schema" "$checkpoint_table" "$source_label" <<'PY' +from pathlib import Path +import sys + +repo_root = Path(sys.argv[1]) +output_file = Path(sys.argv[2]) +catalog, control_schema, bronze_schema, silver_schema, checkpoint_table, source_label = sys.argv[3:] +template_path = repo_root / "platform/databricks/delta/dashboards/convex_sync_overview.lvdash.json.tmpl" +template = template_path.read_text() +rendered = ( + template + .replace("{{CATALOG}}", catalog) + .replace("{{CONTROL_SCHEMA}}", control_schema) + .replace("{{BRONZE_SCHEMA}}", bronze_schema) + .replace("{{SILVER_SCHEMA}}", silver_schema) + .replace("{{CHECKPOINT_TABLE}}", checkpoint_table) + .replace("{{SOURCE_LABEL}}", source_label) +) +output_file.parent.mkdir(parents=True, exist_ok=True) +output_file.write_text(rendered) +print(output_file) +PY From 5fd7eeae5b6926bbf010ff19f084730715e1d49c Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 05:40:49 -0500 Subject: [PATCH 2/2] fix: align dashboard source slug fallback --- scripts/publish-databricks-delta-dashboard.sh | 2 +- scripts/render-databricks-delta-dashboard.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/publish-databricks-delta-dashboard.sh b/scripts/publish-databricks-delta-dashboard.sh index e76b4dc..5d0e25e 100755 --- a/scripts/publish-databricks-delta-dashboard.sh +++ b/scripts/publish-databricks-delta-dashboard.sh @@ -16,7 +16,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "$repo_root/scripts/load-source-config.sh" load_convex_sync_source_config "$repo_root" -source_slug="${CONVEX_SYNC_SOURCE_SLUG:-${CONVEX_SYNC_SOURCE:-default}}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" source_label="${CONVEX_SYNC_SOURCE_LABEL:-$source_slug}" display_name="${DATABRICKS_DELTA_DASHBOARD_NAME:-Convex Sync Overview (${source_label})}" diff --git a/scripts/render-databricks-delta-dashboard.sh b/scripts/render-databricks-delta-dashboard.sh index 5334d64..76d6323 100755 --- a/scripts/render-databricks-delta-dashboard.sh +++ b/scripts/render-databricks-delta-dashboard.sh @@ -14,7 +14,7 @@ source "$repo_root/scripts/load-source-config.sh" load_convex_sync_source_config "$repo_root" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" -source_slug="${CONVEX_SYNC_SOURCE_SLUG:-${CONVEX_SYNC_SOURCE:-default}}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}"