Skip to content

Commit e7452bf

Browse files
author
ci bot
committed
Merge branch 'misc-fixes' into 'enterprise'
Misc UI + SQL fixes See merge request dkinternal/testgen/dataops-testgen!323
2 parents 6ef553a + d060b0e commit e7452bf

18 files changed

Lines changed: 337 additions & 115 deletions

File tree

‎testgen/common/models/scheduler.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from uuid import UUID, uuid4
55

66
from cron_converter import Cron
7-
from sqlalchemy import Column, String, func, select
7+
from sqlalchemy import Boolean, Column, String, delete, func, select, update
88
from sqlalchemy.dialects import postgresql
99
from sqlalchemy.orm import InstrumentedAttribute
1010

@@ -27,14 +27,15 @@ class JobSchedule(Base):
2727
kwargs: dict[str, Any] = Column(postgresql.JSONB, nullable=False, default={})
2828
cron_expr: str = Column(String, nullable=False)
2929
cron_tz: str = Column(String, nullable=False)
30+
active: bool = Column(Boolean, default=True)
3031

3132
@classmethod
3233
def select_where(cls, *clauses, order_by: str | InstrumentedAttribute | None = None) -> Iterable[Self]:
3334
test_definitions_count = (
3435
select(cls.id)
3536
.join(TestSuite, TestSuite.test_suite == cls.kwargs["test_suite_key"].astext)
3637
.join(TestDefinition, TestDefinition.test_suite_id == TestSuite.id)
37-
.where(cls.key == RUN_TESTS_JOB_KEY)
38+
.where(cls.key == RUN_TESTS_JOB_KEY, cls.active == True)
3839
.group_by(cls.id, TestSuite.test_suite)
3940
.having(func.count(TestDefinition.id) > 0)
4041
.subquery()
@@ -44,11 +45,33 @@ def select_where(cls, *clauses, order_by: str | InstrumentedAttribute | None = N
4445
.join(test_definitions_count, test_definitions_count.c.id == cls.id)
4546
.where(*clauses)
4647
)
47-
non_test_runs_query = select(cls).where(cls.key != RUN_TESTS_JOB_KEY, *clauses)
48+
non_test_runs_query = select(cls).where(cls.key != RUN_TESTS_JOB_KEY, cls.active == True, *clauses)
4849
query = test_runs_query.union_all(non_test_runs_query).order_by(order_by)
4950

5051
return get_current_session().execute(query)
5152

53+
@classmethod
54+
def delete(cls, job_id: str | UUID) -> None:
55+
query = delete(cls).where(JobSchedule.id == UUID(job_id))
56+
db_session = get_current_session()
57+
try:
58+
db_session.execute(query)
59+
except ValueError:
60+
db_session.rollback()
61+
else:
62+
db_session.commit()
63+
64+
@classmethod
65+
def update_active(cls, job_id: str | UUID, active: bool) -> None:
66+
query = update(cls).where(JobSchedule.id == UUID(job_id)).values(active=active)
67+
db_session = get_current_session()
68+
try:
69+
db_session.execute(query)
70+
except ValueError:
71+
db_session.rollback()
72+
else:
73+
db_session.commit()
74+
5275
@classmethod
5376
def count(cls):
5477
return get_current_session().query(cls).count()

‎testgen/common/models/table_group.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class TableGroupSummary(EntityMinimal):
4040
latest_profile_start: datetime
4141
latest_profile_table_ct: int
4242
latest_profile_column_ct: int
43+
latest_profile_data_point_ct: int
4344
latest_anomalies_ct: int
4445
latest_anomalies_definite_ct: int
4546
latest_anomalies_likely_ct: int
@@ -123,6 +124,7 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera
123124
latest_run.profiling_starttime,
124125
latest_run.table_ct,
125126
latest_run.column_ct,
127+
latest_run.dq_total_data_points,
126128
latest_run.anomaly_ct,
127129
SUM(
128130
CASE
@@ -172,6 +174,7 @@ def select_summary(cls, project_code: str, for_dashboard: bool = False) -> Itera
172174
latest_profile.profiling_starttime AS latest_profile_start,
173175
latest_profile.table_ct AS latest_profile_table_ct,
174176
latest_profile.column_ct AS latest_profile_column_ct,
177+
latest_profile.dq_total_data_points AS latest_profile_data_point_ct,
175178
latest_profile.anomaly_ct AS latest_anomalies_ct,
176179
latest_profile.definite_ct AS latest_anomalies_definite_ct,
177180
latest_profile.likely_ct AS latest_anomalies_likely_ct,

‎testgen/template/dbsetup/030_initialize_new_schema_structure.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,7 @@ CREATE TABLE job_schedules (
903903
kwargs JSONB NOT NULL,
904904
cron_expr VARCHAR(50) NOT NULL,
905905
cron_tz VARCHAR(30) NOT NULL,
906+
active BOOLEAN DEFAULT TRUE,
906907
UNIQUE (project_code, key, args, kwargs, cron_expr, cron_tz)
907908
);
908909

‎testgen/template/dbsetup_test_types/test_types_Valid_US_Zip3.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,23 +119,23 @@ test_types:
119119
sql_flavor: databricks
120120
lookup_type: null
121121
lookup_query: |-
122-
SELECT `{COLUMN_NAME}`, COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE(`{COLUMN_NAME}`,'012345678','999999999') NOT IN ('99999', '999999999', '99999-9999') GROUP BY `{COLUMN_NAME}` ORDER BY record_ct DESC LIMIT 20;
122+
SELECT `{COLUMN_NAME}`, COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE(`{COLUMN_NAME}`,'012345678','999999999') <> '999' GROUP BY `{COLUMN_NAME}` ORDER BY record_ct DESC LIMIT 20;
123123
error_type: Test Results
124124
- id: '1243'
125125
test_id: '1045'
126126
test_type: Valid_US_Zip3
127127
sql_flavor: mssql
128128
lookup_type: null
129129
lookup_query: |-
130-
SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",'012345678','999999999') NOT IN ('99999', '999999999', '99999-9999') GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;
130+
SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",'012345678','999999999') <> '999' GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;
131131
error_type: Test Results
132132
- id: '1242'
133133
test_id: '1045'
134134
test_type: Valid_US_Zip3
135135
sql_flavor: postgresql
136136
lookup_type: null
137137
lookup_query: |-
138-
SELECT "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",'012345678','999999999') NOT IN ('99999', '999999999', '99999-9999') <> '' GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC LIMIT 20;
138+
SELECT "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",'012345678','999999999') <> '999' GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC LIMIT 20;
139139
error_type: Test Results
140140
- id: '1241'
141141
test_id: '1045'
@@ -159,6 +159,6 @@ test_types:
159159
sql_flavor: snowflake
160160
lookup_type: null
161161
lookup_query: |-
162-
SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",'012345678','999999999') NOT IN ('99999', '999999999', '99999-9999') GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;
162+
SELECT TOP 20 "{COLUMN_NAME}", COUNT(*) AS record_ct FROM {TARGET_SCHEMA}.{TABLE_NAME} WHERE TRANSLATE("{COLUMN_NAME}",'012345678','999999999') <> '999' GROUP BY "{COLUMN_NAME}" ORDER BY record_ct DESC;
163163
error_type: Test Results
164164
test_templates: []

‎testgen/template/dbupgrade/0154_incremental_upgrade.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
SET SEARCH_PATH TO {SCHEMA_NAME};
2+
3+
ALTER TABLE job_schedules ADD COLUMN active BOOLEAN DEFAULT TRUE;

‎testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date
1414
AND ts.id = '{TEST_SUITE_ID}'
1515
AND p.run_date::DATE <= '{AS_OF_DATE}'
1616
GROUP BY r.table_groups_id),
17-
curprof AS (SELECT p.profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type,
17+
curprof AS (SELECT p.profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, column_type,
1818
distinct_value_ct, record_ct, max_value, min_value, avg_value, stdev_value, null_value_ct
1919
FROM last_run lr
2020
INNER JOIN profile_results p
@@ -28,7 +28,7 @@ locked AS (SELECT schema_name, table_name
2828
AND lock_refresh = 'Y'),
2929
-- IDs - TOP 2
3030
id_cols
31-
AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type,
31+
AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, column_type,
3232
distinct_value_ct,
3333
ROW_NUMBER() OVER (PARTITION BY schema_name, table_name
3434
ORDER BY
@@ -42,7 +42,7 @@ id_cols
4242
AND functional_data_type ILIKE 'ID%'),
4343
-- Process Date - TOP 1
4444
process_date_cols
45-
AS (SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type,
45+
AS (SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, column_type,
4646
distinct_value_ct,
4747
ROW_NUMBER() OVER (PARTITION BY schema_name, table_name
4848
ORDER BY
@@ -57,7 +57,7 @@ process_date_cols
5757
AND functional_data_type ILIKE 'process%'),
5858
-- Transaction Date - TOP 1
5959
tran_date_cols
60-
AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type,
60+
AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, column_type,
6161
distinct_value_ct,
6262
ROW_NUMBER() OVER (PARTITION BY schema_name, table_name
6363
ORDER BY
@@ -70,7 +70,7 @@ tran_date_cols
7070

7171
-- Numeric Measures
7272
numeric_cols
73-
AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type,
73+
AS ( SELECT profile_run_id, schema_name, table_name, column_name, functional_data_type, general_type, column_type,
7474
/*
7575
-- Subscores
7676
distinct_value_ct * 1.0 / NULLIF(record_ct, 0) AS cardinality_score,
@@ -98,19 +98,19 @@ numeric_cols_ranked
9898
FROM numeric_cols
9999
WHERE change_detection_score IS NOT NULL),
100100
combined
101-
AS ( SELECT profile_run_id, schema_name, table_name, column_name, 'ID' AS element_type, general_type, 10 + rank AS fingerprint_order
101+
AS ( SELECT profile_run_id, schema_name, table_name, column_name, 'ID' AS element_type, general_type, column_type, 10 + rank AS fingerprint_order
102102
FROM id_cols
103103
WHERE rank <= 2
104104
UNION ALL
105-
SELECT profile_run_id, schema_name, table_name, column_name, 'DATE_P' AS element_type, general_type, 20 + rank AS fingerprint_order
105+
SELECT profile_run_id, schema_name, table_name, column_name, 'DATE_P' AS element_type, general_type, column_type, 20 + rank AS fingerprint_order
106106
FROM process_date_cols
107107
WHERE rank = 1
108108
UNION ALL
109-
SELECT profile_run_id, schema_name, table_name, column_name, 'DATE_T' AS element_type, general_type, 30 + rank AS fingerprint_order
109+
SELECT profile_run_id, schema_name, table_name, column_name, 'DATE_T' AS element_type, general_type, column_type, 30 + rank AS fingerprint_order
110110
FROM tran_date_cols
111111
WHERE rank = 1
112112
UNION ALL
113-
SELECT profile_run_id, schema_name, table_name, column_name, 'MEAS' AS element_type, general_type, 40 + rank AS fingerprint_order
113+
SELECT profile_run_id, schema_name, table_name, column_name, 'MEAS' AS element_type, general_type, column_type, 40 + rank AS fingerprint_order
114114
FROM numeric_cols_ranked
115115
WHERE rank = 1 ),
116116
newtests AS (
@@ -123,7 +123,8 @@ newtests AS (
123123
CASE
124124
WHEN general_type = 'D' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(COUNT(DISTINCT @@@) AS NVARCHAR)'
125125
WHEN general_type = 'A' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(COUNT(DISTINCT @@@) AS NVARCHAR) + ''|'' + CAST(SUM(LEN(@@@)) AS NVARCHAR)'
126-
WHEN general_type = 'N' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(SUM(@@@) AS NVARCHAR) + ''|'' + CAST(ROUND(AVG(@@@), 5) AS NVARCHAR) + ''|'' + CAST(ROUND(STDEV(CAST(@@@ AS FLOAT)), 5) AS NVARCHAR)'
126+
WHEN general_type = 'N' AND column_type ILIKE '%int%' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(SUM(CAST(@@@ AS BIGINT)) AS NVARCHAR) + ''|'' + CAST(ROUND(AVG(CAST(@@@ AS DECIMAL(30,5))), 5) AS NVARCHAR) + ''|'' + CAST(ROUND(STDEV(CAST(@@@ AS FLOAT)), 5) AS NVARCHAR)'
127+
WHEN general_type = 'N' AND column_type NOT ILIKE '%int%' THEN 'CAST(MIN(@@@) AS NVARCHAR) + ''|'' + MAX(CAST(@@@ AS NVARCHAR)) + ''|'' + CAST(SUM(@@@) AS NVARCHAR) + ''|'' + CAST(ROUND(AVG(@@@), 5) AS NVARCHAR) + ''|'' + CAST(ROUND(STDEV(CAST(@@@ AS FLOAT)), 5) AS NVARCHAR)'
127128
END,
128129
'@@@', '"' || column_name || '"'
129130
),

‎testgen/ui/assets/style.css‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,10 @@ div[data-testid="stVerticalBlockBorderWrapper"]:has( > div > div[data-testid="st
253253
justify-content: center;
254254
}
255255

256+
.stVerticalBlock:has(> div.stElementContainer > div.stHtml > i.flex-end) {
257+
flex-wrap: wrap;
258+
}
259+
256260
.stVerticalBlock:has(> div.stElementContainer > div.stHtml > i.no-flex-gap) {
257261
gap: unset;
258262
}
@@ -439,6 +443,45 @@ div[data-testid="stPopoverBody"]:has(i.tg-header--help-wrapper) {
439443
}
440444
/* */
441445

446+
/* Summary counts component */
447+
.tg-summary-counts--label {
448+
margin-bottom: 4px;
449+
}
450+
451+
.tg-summary-counts {
452+
height: 100%;
453+
display: flex;
454+
flex-flow: row nowrap;
455+
align-items: flex-start;
456+
justify-content: flex-start;
457+
gap: 16px;
458+
}
459+
460+
.tg-summary-counts--item {
461+
display: flex;
462+
flex-flow: row nowrap;
463+
align-items: stretch;
464+
gap: 8px;
465+
}
466+
467+
.tg-summary-counts--bar {
468+
width: 4px;
469+
}
470+
471+
.tg-summary-counts--value {
472+
line-height: 1.2;
473+
}
474+
475+
.tg-summary-counts--value > div:first-child {
476+
color: var(--caption-text-color);
477+
font-size: 12px;
478+
}
479+
480+
.tg-summary-counts--value > div:last-child {
481+
font-size: 16px;
482+
}
483+
/* */
484+
442485
/* Export Menu */
443486
.st-key-tg--export-popover [data-testid="stPopoverButton"] > div:last-child {
444487
display: none;

‎testgen/ui/components/frontend/js/components/crontab_input.js‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
* @typedef CronSample
99
* @type {object}
1010
* @property {string?} error
11-
* @property {string?} sample
11+
* @property {string[]?} samples
1212
* @property {string?} readable_expr
1313
*
1414
* @typedef InitialValue
@@ -85,8 +85,8 @@ const CrontabInput = (/** @type Options */ props) => {
8585
}),
8686
),
8787
Portal(
88-
{target: domId.val, align: 'right', style: 'width: 450px;', opened},
89-
() => ContabEditorPortal(
88+
{target: domId.val, align: 'right', style: 'width: 500px;', opened},
89+
() => CrontabEditorPortal(
9090
{
9191
onChange: onEditorChange,
9292
onClose: () => opened.val = false,
@@ -103,7 +103,7 @@ const CrontabInput = (/** @type Options */ props) => {
103103
* @param {import('../van.min.js').VanState<string>} expr
104104
* @returns {HTMLElement}
105105
*/
106-
const ContabEditorPortal = ({sample, ...options}, expr) => {
106+
const CrontabEditorPortal = ({sample, ...options}, expr) => {
107107
const mode = van.state(expr.rawVal ? determineMode(expr.rawVal) : 'x_hours');
108108

109109
const xHoursState = {
@@ -286,11 +286,6 @@ const ContabEditorPortal = ({sample, ...options}, expr) => {
286286
{ class: () => `${mode.val === 'certain_days' ? '' : 'hidden'}`},
287287
div(
288288
{class: 'flex-row fx-gap-2 mb-2'},
289-
Checkbox({
290-
label: 'Sunday',
291-
checked: certainDaysState.sunday,
292-
onChange: (v) => certainDaysState.sunday.val = v,
293-
}),
294289
Checkbox({
295290
label: 'Monday',
296291
checked: certainDaysState.monday,
@@ -301,22 +296,20 @@ const ContabEditorPortal = ({sample, ...options}, expr) => {
301296
checked: certainDaysState.tuesday,
302297
onChange: (v) => certainDaysState.tuesday.val = v,
303298
}),
304-
),
305-
div(
306-
{class: 'flex-row fx-gap-2 mb-2'},
307299
Checkbox({
308300
label: 'Wednesday',
309301
checked: certainDaysState.wednesday,
310302
onChange: (v) => certainDaysState.wednesday.val = v,
311303
}),
304+
),
305+
div(
306+
{class: 'flex-row fx-gap-2 mb-2'},
307+
312308
Checkbox({
313309
label: 'Thursday',
314310
checked: certainDaysState.thursday,
315311
onChange: (v) => certainDaysState.thursday.val = v,
316312
}),
317-
),
318-
div(
319-
{class: 'flex-row fx-gap-2 mb-2'},
320313
Checkbox({
321314
label: 'Friday',
322315
checked: certainDaysState.friday,
@@ -327,6 +320,11 @@ const ContabEditorPortal = ({sample, ...options}, expr) => {
327320
checked: certainDaysState.saturday,
328321
onChange: (v) => certainDaysState.saturday.val = v,
329322
}),
323+
Checkbox({
324+
label: 'Sunday',
325+
checked: certainDaysState.sunday,
326+
onChange: (v) => certainDaysState.sunday.val = v,
327+
}),
330328
),
331329
div(
332330
{class: 'flex-row fx-gap-2'},
@@ -370,10 +368,17 @@ const ContabEditorPortal = ({sample, ...options}, expr) => {
370368
span({class: 'fx-flex'}, ''),
371369
div(
372370
{class: 'flex-column fx-gap-1 mt-3 text-secondary'},
373-
() => span({}, `Cron Expression: ${expr.val ?? ''}`),
374-
() => span({}, `Next Run: ${(getValue(sample) ?? {})?.sample ?? ''}`),
371+
() => span(
372+
{ class: mode.val === 'custom' ? 'hidden': '' },
373+
`Cron Expression: ${expr.val ?? ''}`,
374+
),
375+
() => div(
376+
{ class: 'flex-column' },
377+
span('Next Runs:'),
378+
(getValue(sample) ?? {})?.samples?.map(item => span({ class: 'text-caption' }, item)),
379+
),
375380
() => div(
376-
{class: 'flex-row fx-gap-1 text-caption'},
381+
{class: `flex-row fx-gap-1 text-caption ${mode.val === 'custom' ? '': 'hidden'}`},
377382
span({}, 'Learn more about'),
378383
Link({
379384
open_new: true,

0 commit comments

Comments
 (0)