Merge pull request #59 from DataKitchen/release/5.0.2

Release/5.0.2

Author: datakitchen-devops

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "5.0.1"
+version = "5.0.2"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },

testgen/common/models/test_definition.py

@@ -349,7 +349,7 @@ def copy(
         target_table_name: str | None = None,
         target_column_name: str | None = None,
     ) -> None:
-        modified_columns = [cls.table_groups_id, cls.profile_run_id, cls.test_suite_id]
+        modified_columns = [cls.table_groups_id, cls.profile_run_id, cls.test_suite_id, cls.last_auto_gen_date]
 
         select_columns = [
             literal(target_table_group_id).label("table_groups_id"),
@@ -358,6 +358,7 @@ def copy(
                 else_=None,
             ).label("profile_run_id"),
             literal(target_test_suite_id).label("test_suite_id"),
+            literal(None).label("last_auto_gen_date"),
         ]
 
         if target_table_name:

testgen/template/dbupgrade/0169_incremental_upgrade.sql

@@ -20,6 +20,50 @@ FROM test_results r
 WHERE d.last_auto_gen_date IS NOT NULL
   AND test_results.id = r.id;
 
+-- Handle duplicates before creating unique indexes
+
+UPDATE test_definitions
+  SET last_auto_gen_date = NULL
+WHERE last_auto_gen_date IS NOT NULL
+  AND table_name IS NULL
+  AND column_name IS NULL
+  AND id NOT IN (
+    SELECT DISTINCT ON (test_suite_id, test_type, schema_name) id
+    FROM test_definitions
+    WHERE last_auto_gen_date IS NOT NULL
+      AND table_name IS NULL
+      AND column_name IS NULL
+    ORDER BY test_suite_id, test_type, schema_name, last_auto_gen_date
+  );
+
+UPDATE test_definitions
+  SET last_auto_gen_date = NULL
+WHERE last_auto_gen_date IS NOT NULL
+  AND table_name IS NOT NULL
+  AND column_name IS NULL
+  AND id NOT IN (
+    SELECT DISTINCT ON (test_suite_id, test_type, schema_name, table_name) id
+    FROM test_definitions
+    WHERE last_auto_gen_date IS NOT NULL
+      AND table_name IS NOT NULL
+      AND column_name IS NULL
+    ORDER BY test_suite_id, test_type, schema_name, table_name, last_auto_gen_date
+  );
+
+UPDATE test_definitions
+  SET last_auto_gen_date = NULL
+WHERE last_auto_gen_date IS NOT NULL
+  AND table_name IS NOT NULL
+  AND column_name IS NOT NULL
+  AND id NOT IN (
+    SELECT DISTINCT ON (test_suite_id, test_type, schema_name, table_name, column_name) id
+      FROM test_definitions
+    WHERE last_auto_gen_date IS NOT NULL
+      AND table_name IS NOT NULL
+      AND column_name IS NOT NULL
+    ORDER BY test_suite_id, test_type, schema_name, table_name, column_name, last_auto_gen_date
+  );
+
 CREATE UNIQUE INDEX uix_td_autogen_schema
   ON test_definitions (test_suite_id, test_type, schema_name)
   WHERE last_auto_gen_date IS NOT NULL