fix(monitors): handle dropped and re-added tables

Author: aarthy-dk

testgen/template/data_chars/data_chars_update.sql

@@ -17,20 +17,37 @@ WITH new_chars AS (
       schema_name,
       table_name,
       run_date
+),
+updated_records AS (
+   UPDATE data_table_chars
+   SET approx_record_ct = n.approx_record_ct,
+      record_ct = n.record_ct,
+      column_ct = n.column_ct,
+      last_refresh_date = n.run_date,
+      drop_date = NULL
+   FROM new_chars n
+      INNER JOIN data_table_chars d ON (
+         n.table_groups_id = d.table_groups_id
+         AND n.schema_name = d.schema_name
+         AND n.table_name = d.table_name
+      )
+   WHERE data_table_chars.table_id = d.table_id
+   RETURNING data_table_chars.*, d.drop_date as old_drop_date
+)
+INSERT INTO data_structure_log (
+   table_groups_id,
+   table_id,
+   table_name,
+   change_date,
+   change
 )
-UPDATE data_table_chars
-SET approx_record_ct = n.approx_record_ct,
-   record_ct = n.record_ct,
-   column_ct = n.column_ct,
-   last_refresh_date = n.run_date,
-   drop_date = NULL
-FROM new_chars n
-   INNER JOIN data_table_chars d ON (
-      n.table_groups_id = d.table_groups_id
-      AND n.schema_name = d.schema_name
-      AND n.table_name = d.table_name
-   )
-WHERE data_table_chars.table_id = d.table_id;
+SELECT u.table_groups_id,
+   u.table_id,
+   u.table_name,
+   u.last_refresh_date,
+   'A'
+   FROM updated_records u
+   WHERE u.old_drop_date IS NOT NULL;
 
 -- Add new records
 WITH new_chars AS (
@@ -170,7 +187,7 @@ update_chars AS (
       )
    WHERE data_column_chars.table_id = d.table_id
       AND data_column_chars.column_name = d.column_name
-   RETURNING data_column_chars.*, d.db_data_type as old_data_type
+   RETURNING data_column_chars.*, d.db_data_type as old_data_type, d.drop_date as old_drop_date, n.run_date as run_date
 )
 INSERT INTO data_structure_log (
    table_groups_id,
@@ -193,7 +210,20 @@ SELECT u.table_groups_id,
    u.old_data_type,
    u.db_data_type
    FROM update_chars u
-   WHERE u.old_data_type <> u.db_data_type;
+   WHERE u.old_data_type <> u.db_data_type
+      AND u.old_drop_date IS NULL
+UNION ALL
+SELECT u.table_groups_id,
+   u.table_id,
+   u.column_id,
+   u.table_name,
+   u.column_name,
+   u.run_date,
+   'A',
+   NULL,
+   u.db_data_type
+   FROM update_chars u
+   WHERE u.old_drop_date IS NOT NULL;
 
 
 -- Add new records

testgen/ui/components/frontend/js/pages/table_monitoring_trends.js

@@ -108,12 +108,27 @@ const TableMonitoringTrend = (props) => {
   ], []);
   const freshnessEvents = (getValue(props.freshness_events) ?? []).map(e => ({ ...e, time: parseDate(e.time) }));
   const schemaChangeEvents = (getValue(props.schema_events) ?? []).map(e => ({ ...e, time: parseDate(e.time), window_start: parseDate(e.window_start) }));
-  const volumeTrendEvents = (getValue(props.volume_events) ?? []).map(e => ({ ...e, time: parseDate(e.time) }));
   const schemaChangesMaxValue = schemaChangeEvents.reduce((currentValue, e) => Math.max(currentValue, e.additions, e.deletions), 10);
 
+  // Compute dropped periods from schema events to hide volume/metric data between table drop and re-add
+  const droppedPeriods = [];
+  let dropStart = null;
+  const sorted = [...schemaChangeEvents].sort((a, b) => a.time - b.time);
+  for (const event of sorted) {
+    if (event.table_change === 'D' && dropStart === null) {
+      dropStart = event.time;
+    } else if (event.table_change === 'A' && dropStart !== null) {
+      droppedPeriods.push({ start: dropStart, end: event.time });
+      dropStart = null;
+    }
+  }
+  const isInDroppedPeriod = (time) => droppedPeriods.some(p => time >= p.start && time <= p.end);
+
+  const volumeTrendEvents = (getValue(props.volume_events) ?? []).map(e => ({ ...e, time: parseDate(e.time) })).filter(e => !isInDroppedPeriod(e.time));
+
   const metricEventGroups = metricEvents.map(group => ({
     ...group,
-    events: group.events.map(e => ({ ...e, time: parseDate(e.time) })),
+    events: group.events.map(e => ({ ...e, time: parseDate(e.time) })).filter(e => !isInDroppedPeriod(e.time)),
   }));
 
   const volumes = [

testgen/ui/views/monitors_dashboard.py

@@ -834,6 +834,7 @@ def get_monitor_events_for_table(test_suite_id: str, table_name: str) -> dict:
         ],
         "schema_events": [
             {
+                "table_change": signals[0] or None,
                 "additions": signals[1],
                 "deletions": signals[2],
                 "modifications": signals[3],