Reusable rules (GoogleCloudPlatform#17232)

XianJiChen · web-flow · commit d9e3c722e878 · 2026-04-28T21:11:42.000Z
diff --git a/mmv1/products/dataplex/Datascan.yaml b/mmv1/products/dataplex/Datascan.yaml
@@ -175,6 +175,20 @@ examples:
       project_name: 'PROJECT_NAME'
       location: 'REGION'
     external_providers: ["time"]
+  - name: 'dataplex_datascan_quality_reusable_rules_catalog_based'
+    primary_resource_id: 'reusable_rules_catalog_based'
+    vars:
+      datascan_name: 'dataquality-catalog'
+    test_env_vars:
+      project_name: 'PROJECT_NAME'
+    external_providers: ["time"]
+  - name: 'dataplex_datascan_data_quality_template_reference'
+    primary_resource_id: 'data_quality_template_reference'
+    vars:
+      datascan_name: 'dataquality-template'
+    test_env_vars:
+      project_name: 'PROJECT_NAME'
+    external_providers: ["time"]
 parameters:
   - name: 'location'
     type: String
@@ -506,6 +520,10 @@ properties:
               description: |
                 Description of the rule.
                 The maximum length is 1,024 characters.
+            - name: 'attributes'
+              type: KeyValuePairs
+              description: |
+                Map of attribute name and value linked to the rule.
             - name: 'rangeExpectation'
               type: NestedObject
               description: |
@@ -636,11 +654,42 @@ properties:
                   description: |
                     The SQL statement.
                   required: true
+            - name: 'templateReference'
+              type: NestedObject
+              description: |
+                Aggregate rule which references a rule template and provides the parameters to be substituted in the template.
+              properties:
+                - name: 'name'
+                  type: String
+                  description: |
+                    The resource name of the template entry.
+                  required: true
+                - name: 'values'
+                  type: Map
+                  key_name: 'name'
+                  description: |
+                    The map of parameter name and value.
+                  value_type:
+                    type: NestedObject
+                    properties:
+                      - name: 'value'
+                        type: String
+                        required: true
+                        description: |
+                          The string representation of the parameter value.
         min_size: 1
       - name: 'catalogPublishingEnabled'
         type: Boolean
         description: |
           If set, the latest DataScan job result will be published to Dataplex Catalog.
+      - name: 'enableCatalogBasedRules'
+        type: Boolean
+        description: |
+          If set to true, the scan will retrieve rules defined in Data Catalog for the resource.
+      - name: 'filter'
+        type: String
+        description: |
+          A filter to selectively run a subset of rules.
   - name: 'dataProfileSpec'
     type: NestedObject
     description: |
diff --git a/mmv1/templates/terraform/examples/dataplex_datascan_data_quality_template_reference.tf.tmpl b/mmv1/templates/terraform/examples/dataplex_datascan_data_quality_template_reference.tf.tmpl
@@ -0,0 +1,133 @@
+data "google_project" "project" {
+  project_id = "{{index $.TestEnvVars "project_name"}}"
+}
+
+resource "google_service_account" "sa" {
+  account_id   = "tf-test-sa-%{random_suffix}"
+  display_name = "DataScan Service Account"
+  project      = data.google_project.project.project_id
+}
+
+resource "google_service_account_iam_member" "dataplex_sa_impersonate" {
+  service_account_id = google_service_account.sa.name
+  role               = "roles/iam.serviceAccountTokenCreator"
+  member             = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-dataplex.iam.gserviceaccount.com"
+}
+resource "time_sleep" "wait_120_seconds" {
+  depends_on = [google_service_account_iam_member.dataplex_sa_impersonate]
+  create_duration = "120s"
+}
+
+
+resource "google_project_iam_member" "sa_bq_data_viewer" {
+  project = data.google_project.project.project_id
+  role    = "roles/bigquery.dataViewer"
+  member  = "serviceAccount:${google_service_account.sa.email}"
+}
+
+resource "google_project_iam_member" "sa_bq_job_user" {
+  project = data.google_project.project.project_id
+  role    = "roles/bigquery.jobUser"
+  member  = "serviceAccount:${google_service_account.sa.email}"
+}
+
+resource "google_dataplex_entry_group" "test_group" {
+  location       = "us-central1"
+  entry_group_id = "test-group-%{random_suffix}"
+  project        = data.google_project.project.project_id
+}
+
+resource "google_dataplex_entry" "test_entry" {
+  location       = "us-central1"
+  entry_group_id = google_dataplex_entry_group.test_group.entry_group_id
+  entry_id       = "test-entry-%{random_suffix}"
+  entry_type     = "projects/655216118709/locations/global/entryTypes/data-quality-rule-template"
+  project        = data.google_project.project.number
+  aspects {
+    aspect_key = "655216118709.global.data-quality-rule-template"
+    aspect {
+      data = jsonencode({
+        dimension = "VALIDITY"
+        sqlCollection = [
+          {
+            query = "SELECT * FROM $${data()} WHERE $${column()} IS NOT NULL"
+          }
+        ]
+      })
+    }
+  }
+}
+
+resource "google_bigquery_dataset" "tf_test_dataset" {
+  dataset_id = "tf_test_dataset_id_%{random_suffix}"
+  default_table_expiration_ms = 3600000
+  location   = "us-central1"
+  project    = data.google_project.project.project_id
+
+  depends_on = [
+    google_service_account_iam_member.dataplex_sa_impersonate,
+    google_project_iam_member.sa_bq_data_viewer,
+    google_project_iam_member.sa_bq_job_user
+  ]
+}
+
+resource "google_bigquery_table" "tf_test_table" {
+  dataset_id          = google_bigquery_dataset.tf_test_dataset.dataset_id
+  table_id            = "tf_test_table_id_%{random_suffix}"
+  deletion_protection = false
+  project             = data.google_project.project.project_id
+  schema              = <<EOF
+    [
+    {
+      "name": "name",
+      "type": "STRING",
+      "mode": "NULLABLE"
+    }
+    ]
+  EOF
+}
+
+
+resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
+  location = "us-central1"
+  display_name = "Data Quality Template Reference"
+  data_scan_id = "{{index $.Vars "datascan_name"}}"
+
+  data {
+    resource = "//bigquery.googleapis.com/projects/${data.google_project.project.project_id}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/${google_bigquery_table.tf_test_table.table_id}"
+  }
+
+  execution_spec {
+    trigger {
+      on_demand {}
+    }
+  }
+
+  execution_identity {
+    service_account {
+      email = google_service_account.sa.email
+    }
+  }
+
+  data_quality_spec {
+    rules {
+      column = "name"
+      dimension = "VALIDITY"
+      template_reference {
+        name = google_dataplex_entry.test_entry.name
+        values {
+          name = "min_length"
+          value = "10"
+        }
+      }
+    }
+  }
+
+
+  project = data.google_project.project.project_id
+
+  depends_on = [
+    google_bigquery_table.tf_test_table,
+    time_sleep.wait_120_seconds
+  ]
+}
diff --git a/mmv1/templates/terraform/examples/dataplex_datascan_full_quality.tf.tmpl b/mmv1/templates/terraform/examples/dataplex_datascan_full_quality.tf.tmpl
@@ -24,6 +24,7 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
     sampling_percent = 5
     row_filter = "station_id > 1000"
     catalog_publishing_enabled = true
+    filter = "attributes.priority = 'high'"
     post_scan_actions {
       notification_report {
         recipients {
@@ -39,6 +40,9 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
       column = "address"
       dimension = "VALIDITY"
       threshold = 0.99
+      attributes = {
+        priority = "high"
+      }
       non_null_expectation {}
     }
 
diff --git a/mmv1/templates/terraform/examples/dataplex_datascan_full_quality_test.tf.tmpl b/mmv1/templates/terraform/examples/dataplex_datascan_full_quality_test.tf.tmpl
@@ -1,6 +1,7 @@
 resource "google_bigquery_dataset" "tf_test_dataset" {
   dataset_id = "tf_test_dataset_id_%{random_suffix}"
   default_table_expiration_ms = 3600000
+  location   = "us-central1"
 }
 
 resource "google_bigquery_table" "tf_test_table" {
@@ -71,7 +72,7 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
   }
 
   data {
-    resource = "//bigquery.googleapis.com/projects/{{index $.TestEnvVars "project_name"}}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/${google_bigquery_table.tf_test_table.table_id}"
+    resource = "//bigquery.googleapis.com/projects/${"{{index $.TestEnvVars "project_name"}}"}/datasets/${google_bigquery_dataset.tf_test_dataset.dataset_id}/tables/${google_bigquery_table.tf_test_table.table_id}"
   }
 
   execution_spec {
@@ -86,6 +87,7 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
     sampling_percent = 5
     row_filter = "station_id > 1000"
     catalog_publishing_enabled = true
+    filter = "attributes.priority = \"high\""
     post_scan_actions {
       notification_report {
         recipients {
@@ -101,6 +103,9 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
       column = "address"
       dimension = "VALIDITY"
       threshold = 0.99
+      attributes = {
+        priority = "high"
+      }
       non_null_expectation {}
     }
 
@@ -109,6 +114,9 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
       dimension = "VALIDITY"
       ignore_null = true
       threshold = 0.9
+      attributes = {
+        priority = "low"
+      }
       range_expectation {
         min_value = 1
         max_value = 10
@@ -121,6 +129,9 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
       column = "power_type"
       dimension = "VALIDITY"
       ignore_null = false
+      attributes = {
+        priority = "high"
+      }
       regex_expectation {
         regex = ".*solar.*"
       }
@@ -130,6 +141,9 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
       column = "property_type"
       dimension = "VALIDITY"
       ignore_null = false
+      attributes = {
+        priority = "low"
+      }
       set_expectation {
         values = ["sidewalk", "parkland"]
       }
@@ -139,12 +153,18 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
     rules {
       column = "address"
       dimension = "UNIQUENESS"
+      attributes = {
+        priority = "high"
+      }
       uniqueness_expectation {}
     }
 
     rules {
       column = "number_of_docks"
       dimension = "VALIDITY"
+      attributes = {
+        priority = "low"
+      }
       statistic_range_expectation {
         statistic = "MEAN"
         min_value = 5
@@ -157,20 +177,29 @@ resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
     rules {
       column = "footprint_length"
       dimension = "VALIDITY"
+      attributes = {
+        priority = "high"
+      }
       row_condition_expectation {
         sql_expression = "footprint_length > 0 AND footprint_length <= 10"
       }
     }
 
     rules {
       dimension = "VALIDITY"
+      attributes = {
+        priority = "low"
+      }
       table_condition_expectation {
         sql_expression = "COUNT(*) > 0"
       }
     }
 
     rules {
       dimension = "VALIDITY"
+      attributes = {
+        priority = "high"
+      }
       sql_assertion {
         sql_statement = "select * from $${data()} where address is null"
       }
diff --git a/mmv1/templates/terraform/examples/dataplex_datascan_quality_reusable_rules_catalog_based.tf.tmpl b/mmv1/templates/terraform/examples/dataplex_datascan_quality_reusable_rules_catalog_based.tf.tmpl
diff --git a/mmv1/third_party/terraform/services/dataplex/resource_dataplex_datascan_test.go b/mmv1/third_party/terraform/services/dataplex/resource_dataplex_datascan_test.go
