Skip to content

Commit beb6749

Browse files
author
ci bot
committed
Merge branch 'aarthy/redshift-freshness' into 'enterprise'
fix(freshness): numeric overflow errors See merge request dkinternal/testgen/dataops-testgen!402
2 parents 563bb16 + 50666bb commit beb6749

7 files changed

Lines changed: 42 additions & 32 deletions

File tree

‎testgen/common/notifications/monitor_run.py‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class MonitorEmailTemplate(BaseNotificationTemplate):
2222
def get_subject_template(self) -> str:
2323
return (
2424
"[TestGen] Monitors Alert: {{summary.table_groups_name}}"
25-
"{{#if summary.table_name}} | {{test_run_monitor_summary.table_name}}{{/if}}"
25+
"{{#if summary.table_name}} | {{summary.table_name}}{{/if}}"
2626
' | {{total_anomalies}} {{pluralize total_anomalies "anomaly" "anomalies"}}'
2727
)
2828

@@ -65,7 +65,7 @@ def get_main_content_template(self):
6565
border="0">
6666
<tr>
6767
<td class="summary__title">Anomalies Summary</td>
68-
<td align="left">
68+
<td align="right">
6969
<a class="link" href="{{view_in_testgen_url}}" target="_blank">View on TestGen &gt;</a>
7070
</td>
7171
</tr>
@@ -94,7 +94,7 @@ def get_main_content_template(self):
9494
border="0">
9595
<tr class="text-caption">
9696
<td>Table</td>
97-
<td>Type/Focus</td>
97+
<td>Monitor</td>
9898
<td>Details</td>
9999
</tr>
100100
{{#each anomalies}}
@@ -119,19 +119,18 @@ def get_main_content_template(self):
119119

120120
def get_anomaly_tag_template(self):
121121
return """
122-
<td valign="middle">
122+
<td valign="middle" style="padding-right: 24px;">
123123
<table border="0" cellpadding="0" cellspacing="0" role="presentation">
124124
<tr>
125125
<td valign="middle">
126126
<div style="{{#if count}}background-color: #EF5350; min-width: 15px; padding: 0px 5px; border-radius: 10px; text-align: center; line-height: 20px;{{else}}background-color: #9CCC65; width: 20px; height: 20px; border-radius: 50%; text-align: center; line-height: 21px;{{/if}} color: #ffffff; font-weight: bold; font-size: 13px;">
127127
{{#if count}}{{count}}{{else}}&#10003;{{/if}}
128128
</div>
129129
</td>
130-
<td width="4"></td> <td valign="middle" style="color: #111111;">{{type}}</td>
130+
<td valign="middle" style="color: #111111; padding-left: 8px;">{{type}}</td>
131131
</tr>
132132
</table>
133133
</td>
134-
{{#if @last}}{{else}}<td width="16"></td>{{/if}}
135134
"""
136135

137136
def get_extra_css_template(self) -> str:
@@ -171,7 +170,7 @@ def get_extra_css_template(self) -> str:
171170

172171
@log_and_swallow_exception
173172
@with_database_session
174-
def send_monitor_notifications(test_run: TestRun):
173+
def send_monitor_notifications(test_run: TestRun, result_list_ct=20):
175174
notifications = list(MonitorNotificationSettings.select(
176175
enabled=True,
177176
test_suite_id=test_run.test_suite_id,
@@ -206,10 +205,20 @@ def send_monitor_notifications(test_run: TestRun):
206205
for test_result in test_results:
207206
label = _TEST_TYPE_LABELS.get(test_result.test_type)
208207
anomaly_counts[label] = (anomaly_counts.get(label) or 0) + 1
208+
details = test_result.message or "N/A"
209+
210+
if test_result.test_type == "Freshness_Trend":
211+
parts = details.split(". ", 1)
212+
message = parts[1].rstrip(".") if len(parts) > 1 else None
213+
prefix = "Table updated" if "detected: Yes" in details else "No table update"
214+
details = f"{prefix} - {message}" if message else prefix
215+
elif test_result.test_type == "Metric_Trend":
216+
label = f"{label}: {test_result.column_names}"
217+
209218
anomalies.append({
210219
"table_name": test_result.table_name or "N/A",
211220
"type": label,
212-
"details": test_result.message or "N/A",
221+
"details": details,
213222
})
214223

215224
view_in_testgen_url = "".join(
@@ -238,12 +247,8 @@ def send_monitor_notifications(test_run: TestRun):
238247
{"type": key, "count": value}
239248
for key, value in anomaly_counts.items()
240249
],
241-
# "anomaly_counts": [
242-
# {"type": "Freshness", "count": freshness_anomalies},
243-
# {"type": "Schema", "count": schema_anomalies},
244-
# {"type": "Volume", "count": volume_anomalies},
245-
# ],
246-
"anomalies": anomalies,
250+
"anomalies": anomalies[:result_list_ct],
251+
"truncated": max(len(anomalies) - result_list_ct, 0),
247252
"view_in_testgen_url": view_in_testgen_url,
248253
},
249254
)
@@ -252,7 +257,8 @@ def send_monitor_notifications(test_run: TestRun):
252257

253258

254259
_TEST_TYPE_LABELS = {
255-
"Table_Freshness": "Freshness",
256-
"Schema_Drift": "Schema",
260+
"Freshness_Trend": "Freshness",
257261
"Volume_Trend": "Volume",
262+
"Schema_Drift": "Schema",
263+
"Metric_Trend": "Metric",
258264
}

‎testgen/common/notifications/test_run.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -323,15 +323,17 @@ def send_test_run_notifications(test_run: TestRun, result_list_ct=20, result_sta
323323

324324
tr_summary, = TestRun.select_summary(test_run_ids=[test_run.id])
325325

326+
test_run_url = "".join(
327+
(
328+
PersistedSetting.get("BASE_URL", ""),
329+
"/test-runs:results?run_id=",
330+
str(test_run.id),
331+
)
332+
)
333+
326334
context = {
327335
"test_run": tr_summary,
328-
"test_run_url": "".join(
329-
(
330-
PersistedSetting.get("BASE_URL", ""),
331-
"/test-runs:results?run_id=",
332-
str(test_run.id),
333-
)
334-
),
336+
"test_run_url": test_run_url,
335337
"test_run_id": str(test_run.id),
336338
"test_result_summary": [
337339
{
@@ -340,6 +342,7 @@ def send_test_run_notifications(test_run: TestRun, result_list_ct=20, result_sta
340342
"total": test_run.ct_by_status[status],
341343
"truncated": test_run.ct_by_status[status] - len(result_list),
342344
"result_list": result_list,
345+
"test_run_url": test_run_url,
343346
}
344347
for status, label in summary_statuses
345348
if (result_list := result_list_by_status.get(status, None))

‎testgen/template/flavors/databricks/gen_query_tests/gen_Freshness_Trend.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,8 @@ selected_tables AS (
142142
COUNT(DISTINCT MOD((COALESCE(@@@,0)::DECIMAL(38,6) * 1000000)::DECIMAL(38,0), 1000003))::STRING,
143143
COALESCE((MIN(@@@)::DECIMAL(38,6))::STRING, ''''),
144144
COALESCE((MAX(@@@)::DECIMAL(38,6))::STRING, ''''),
145-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000007)), 0), 1000000007)::STRING, ''''),
146-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000009)), 0), 1000000009)::STRING, '''')
145+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000007)), 0), 1000000007)::STRING, ''''),
146+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000009)), 0), 1000000009)::STRING, '''')
147147
)'
148148
END,
149149
'@@@', '`' || column_name || '`'

‎testgen/template/flavors/databricks/gen_query_tests/gen_Table_Freshness.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,8 @@ selected_tables AS (
134134
COUNT(DISTINCT MOD((COALESCE(@@@,0)::DECIMAL(38,6) * 1000000)::DECIMAL(38,0), 1000003))::STRING,
135135
COALESCE((MIN(@@@)::DECIMAL(38,6))::STRING, ''''),
136136
COALESCE((MAX(@@@)::DECIMAL(38,6))::STRING, ''''),
137-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000007)), 0), 1000000007)::STRING, ''''),
138-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000009)), 0), 1000000009)::STRING, '''')
137+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000007)), 0), 1000000007)::STRING, ''''),
138+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000009)), 0), 1000000009)::STRING, '''')
139139
)'
140140
END,
141141
'@@@', '`' || column_name || '`'

‎testgen/template/gen_query_tests/gen_Freshness_Trend.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,8 @@ selected_tables AS (
141141
COUNT(DISTINCT MOD((COALESCE(@@@,0)::DECIMAL(38,6) * 1000000)::DECIMAL(38,0), 1000003))::VARCHAR || ''|'' ||
142142
COALESCE((MIN(@@@)::DECIMAL(38,6))::VARCHAR, '''') || ''|'' ||
143143
COALESCE((MAX(@@@)::DECIMAL(38,6))::VARCHAR, '''') || ''|'' ||
144-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000007)), 0), 1000000007)::VARCHAR, '''') || ''|'' ||
145-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000009)), 0), 1000000009)::VARCHAR, '''')'
144+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000007)), 0), 1000000007)::VARCHAR, '''') || ''|'' ||
145+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000009)), 0), 1000000009)::VARCHAR, '''')'
146146
END,
147147
'@@@', '"' || column_name || '"'
148148
),

‎testgen/template/gen_query_tests/gen_Table_Freshness.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,8 @@ selected_tables AS (
133133
COUNT(DISTINCT MOD((COALESCE(@@@,0)::DECIMAL(38,6) * 1000000)::DECIMAL(38,0), 1000003))::VARCHAR || ''|'' ||
134134
COALESCE((MIN(@@@)::DECIMAL(38,6))::VARCHAR, '''') || ''|'' ||
135135
COALESCE((MAX(@@@)::DECIMAL(38,6))::VARCHAR, '''') || ''|'' ||
136-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000007)), 0), 1000000007)::VARCHAR, '''') || ''|'' ||
137-
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL, 1000000009)), 0), 1000000009)::VARCHAR, '''')'
136+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000007)), 0), 1000000007)::VARCHAR, '''') || ''|'' ||
137+
COALESCE(MOD(COALESCE(SUM(MOD((ABS(COALESCE(@@@,0))::DECIMAL(38,6) * 1000000)::DECIMAL(38,6), 1000000009)), 0), 1000000009)::VARCHAR, '''')'
138138
END,
139139
'@@@', '"' || column_name || '"'
140140
),

‎testgen/ui/views/test_results.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,8 @@ def write_history_chart_v2(data: pd.DataFrame):
665665
return testgen.testgen_component(
666666
"test_results_chart",
667667
props={
668-
"data": data.to_dict("records"),
668+
# Fix NaN values
669+
"data": json.loads(data.to_json(orient="records")),
669670
},
670671
)
671672

0 commit comments

Comments
 (0)