From ab1c3252da2fca3a0676f1947ffc139ab2ca364e Mon Sep 17 00:00:00 2001 From: aasthabharill Date: Wed, 3 Jun 2026 10:50:23 +0000 Subject: [PATCH] initial changes --- .../dataflow_job.tf | 292 ++++++++++++++++++ .../samples/gcs-spanner-dv/README.md | 138 +++++++++ .../terraform/samples/gcs-spanner-dv/main.tf | 54 ++++ .../samples/gcs-spanner-dv/outputs.tf | 9 + .../samples/gcs-spanner-dv/terraform.tf | 42 +++ .../samples/gcs-spanner-dv/terraform.tfvars | 25 ++ .../gcs-spanner-dv/terraform_simple.tfvars | 9 + .../samples/gcs-spanner-dv/variables.tf | 156 ++++++++++ 8 files changed, 725 insertions(+) create mode 100644 v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tfvars create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars create mode 100644 v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf diff --git a/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf new file mode 100644 index 0000000000..fdf5210b92 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/GCS_Spanner_Data_Validator/dataflow_job.tf @@ -0,0 +1,292 @@ + + +# Autogenerated file. DO NOT EDIT. +# +# Copyright (C) 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + + +variable "on_delete" { + type = string + description = "One of \"drain\" or \"cancel\". Specifies behavior of deletion during terraform destroy." +} + +variable "project" { + type = string + description = "The Google Cloud Project ID within which this module provisions resources." +} + +variable "region" { + type = string + description = "The region in which the created job should run." +} + +variable "gcsInputDirectory" { + type = string + description = "This directory is used to read the AVRO files of the records read from source. For example, `gs://your-bucket/your-path`" + default = null +} + +variable "projectId" { + type = string + description = "This is the name of the Cloud Spanner project." + default = null +} + +variable "spannerHost" { + type = string + description = "The Cloud Spanner endpoint to call in the template. For example, `https://batch-spanner.googleapis.com`. Defaults to: https://batch-spanner.googleapis.com." + default = null +} + +variable "instanceId" { + type = string + description = "The destination Cloud Spanner instance." + +} + +variable "databaseId" { + type = string + description = "The destination Cloud Spanner database." + +} + +variable "spannerPriority" { + type = string + description = "The request priority for Cloud Spanner calls. The value must be one of: [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`." + default = null +} + +variable "sessionFilePath" { + type = string + description = "Session file path in Cloud Storage that contains mapping information from Spanner Migration Tool. Defaults to empty." + default = null +} + +variable "schemaOverridesFilePath" { + type = string + description = "A file which specifies the table and the column name overrides from source to spanner. Defaults to empty." + default = null +} + +variable "tableOverrides" { + type = string + description = "These are the table name overrides from source to spanner. They are written in thefollowing format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}]This example shows mapping Singers table to Vocalists and Albums table to Records. For example, `[{Singers, Vocalists}, {Albums, Records}]`. Defaults to empty." + default = null +} + +variable "columnOverrides" { + type = string + description = "These are the column name overrides from source to spanner. They are written in thefollowing format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]Note that the SourceTableName should remain the same in both the source and spanner pair. To override table names, use tableOverrides.The example shows mapping SingerName to TalentName and AlbumName to RecordName in Singers and Albums table respectively. For example, `[{Singers.SingerName, Singers.TalentName}, {Albums.AlbumName, Albums.RecordName}]`. Defaults to empty." + default = null +} + +variable "bigQueryDataset" { + type = string + description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" + +} + +variable "runId" { + type = string + description = "A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`" + default = null +} + +variable "transformationJarPath" { + type = string + description = "Custom jar location in Cloud Storage that contains the custom transformation logic for processing records. Defaults to empty." + default = null +} + +variable "transformationClassName" { + type = string + description = "Fully qualified class name having the custom transformation logic. It is a mandatory field in case transformationJarPath is specified. Defaults to empty." + default = null +} + +variable "transformationCustomParameters" { + type = string + description = "String containing any custom parameters to be passed to the custom transformation class. Defaults to empty." + default = null +} + + +provider "google" { + project = var.project +} + +provider "google-beta" { + project = var.project +} + +variable "additional_experiments" { + type = set(string) + description = "List of experiments that should be used by the job. An example value is 'enable_stackdriver_agent_metrics'." + default = null +} + +variable "autoscaling_algorithm" { + type = string + description = "The algorithm to use for autoscaling" + default = null +} + +variable "enable_streaming_engine" { + type = bool + description = "Indicates if the job should use the streaming engine feature." + default = null +} + +variable "ip_configuration" { + type = string + description = "The configuration for VM IPs. Options are 'WORKER_IP_PUBLIC' or 'WORKER_IP_PRIVATE'." + default = null +} + +variable "kms_key_name" { + type = string + description = "The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY" + default = null +} + +variable "labels" { + type = map(string) + description = "User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. NOTE: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource." + default = null +} + +variable "launcher_machine_type" { + type = string + description = "The machine type to use for launching the job. The default is n1-standard-1." + default = null +} + +variable "machine_type" { + type = string + description = "The machine type to use for the job." + default = null +} + +variable "max_workers" { + type = number + description = "The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000." + default = null +} + +variable "name" { + type = string +} + +variable "network" { + type = string + description = "The network to which VMs will be assigned. If it is not provided, 'default' will be used." + default = null +} + +variable "num_workers" { + type = number + description = "The initial number of Google Compute Engine instances for the job." + default = null +} + +variable "sdk_container_image" { + type = string + description = "Docker registry location of container image to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines." + default = null +} + +variable "service_account_email" { + type = string + description = "The Service Account email used to create the job." + default = null +} + +variable "skip_wait_on_job_termination" { + type = bool + description = "If true, treat DRAINING and CANCELLING as terminal job states and do not wait for further changes before removing from terraform state and moving on. WARNING: this will lead to job name conflicts if you do not ensure that the job names are different, e.g. by embedding a release ID or by using a random_id." + default = null +} + +variable "staging_location" { + type = string + description = "The Cloud Storage path to use for staging files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null +} + +variable "subnetwork" { + type = string + description = "The subnetwork to which VMs will be assigned. Should be of the form 'regions/REGION/subnetworks/SUBNETWORK'." + default = null +} + +variable "temp_location" { + type = string + description = "The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://." + default = null +} + +resource "google_project_service" "required" { + service = "dataflow.googleapis.com" + disable_on_destroy = false +} + +resource "google_dataflow_flex_template_job" "generated" { + depends_on = [google_project_service.required] + provider = google-beta + container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" + parameters = { + gcsInputDirectory = var.gcsInputDirectory + projectId = var.projectId + spannerHost = var.spannerHost + instanceId = var.instanceId + databaseId = var.databaseId + spannerPriority = var.spannerPriority + sessionFilePath = var.sessionFilePath + schemaOverridesFilePath = var.schemaOverridesFilePath + tableOverrides = var.tableOverrides + columnOverrides = var.columnOverrides + bigQueryDataset = var.bigQueryDataset + runId = var.runId + transformationJarPath = var.transformationJarPath + transformationClassName = var.transformationClassName + transformationCustomParameters = var.transformationCustomParameters + } + + additional_experiments = var.additional_experiments + autoscaling_algorithm = var.autoscaling_algorithm + enable_streaming_engine = var.enable_streaming_engine + ip_configuration = var.ip_configuration + kms_key_name = var.kms_key_name + labels = var.labels + launcher_machine_type = var.launcher_machine_type + machine_type = var.machine_type + max_workers = var.max_workers + name = var.name + network = var.network + num_workers = var.num_workers + sdk_container_image = var.sdk_container_image + service_account_email = var.service_account_email + skip_wait_on_job_termination = var.skip_wait_on_job_termination + staging_location = var.staging_location + subnetwork = var.subnetwork + temp_location = var.temp_location + region = var.region +} + +output "dataflow_job_url" { + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.generated.job_id}" +} + diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md new file mode 100644 index 0000000000..8c740f28f8 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/README.md @@ -0,0 +1,138 @@ +## Sample Scenario: GCS Spanner Data Validation + +> **_SCENARIO:_** This Terraform example illustrates launching GCS Spanner Data Validation Dataflow job. + +## Terraform permissions + +In order to create the resources in this sample, +the `Service account`/`User account` being used to run Terraform +should have the +required [permissions](https://cloud.google.com/iam/docs/manage-access-service-accounts#multiple-roles-console). +There are two ways to add permissions - + +1. Adding pre-defined roles to the service account running Terraform. +2. Creating a custom role with the granular permissions and attaching it to the + service account running Terraform. + +### Using custom role and granular permissions (recommended) + +Following permissions are required - + +```shell +- dataflow.jobs.cancel +- dataflow.jobs.create +- dataflow.jobs.updateContents +- iam.roles.get +- iam.serviceAccounts.actAs +- resourcemanager.projects.setIamPolicy +- storage.objects.create +- storage.objects.get +- storage.objects.list +- bigquery.datasets.get +- bigquery.tables.create +- bigquery.tables.get +- bigquery.tables.updateData +- bigquery.jobs.create +- serviceusage.services.use +- serviceusage.services.enable +``` + +**Note**: Add the `roles/viewer` role as well to the service account. + +### Using pre-defined roles + +Following roles are required - + +```shell +roles/dataflow.admin +roles/iam.securityAdmin +roles/iam.serviceAccountUser +roles/storage.admin +roles/bigquery.admin +roles/viewer +``` + +## Dataflow permissions + +The Dataflow service account needs to be provided the `roles/storage.objectViewer`, `roles/spanner.databaseReader`, `roles/bigquery.dataEditor` and `roles/bigquery.jobUser` roles. + +## Assumptions + +It makes the following assumptions - + +1. Appropriate permissions are added to the service account running Terraform to allow resource creation. +2. Appropriate permissions are provided to the service account running Dataflow to read from GCS and Spanner, and write results to BigQuery. +3. GCS directory containing source AVRO files is already present. +4. Destination Spanner instance and database are already created. +5. A BigQuery dataset has been created where the validation results will be stored. + +## Description + +This sample contains the following files - + +1. `main.tf` - This contains the Terraform resources which will be created. +2. `outputs.tf` - This declares the outputs that will be output as part of + running this terraform example. +3. `variables.tf` - This declares the input variables that are required to + configure the resources. +4. `terraform.tf` - This contains the required providers and APIs/project + configurations for this sample. +5. `terraform.tfvars` - This contains the dummy inputs that need to be populated + to run this example. +6. `terraform_simple.tfvars` - This contains the minimal list of dummy inputs + that need to be populated to run this example. + +## Resources Created + +It creates the following resources - + +1. **Dataflow job** - The Dataflow job which reads from GCS and Spanner and compares them. +2. **Permissions** - It adds the required roles to the specified (or the + default) service accounts for the validation job to work. + +## How to run + +1. Clone this repository or the sample locally. +2. Edit the `terraform.tfvars` file and replace the dummy variables with real values. Extend the configuration to meet + your needs. It is recommended to get started with `terraform_simple.tfvars`. +3. Run the following commands - + +### Initialise Terraform + +```shell +# Initialise terraform - You only need to do this once for a directory. +terraform init +``` + +### Run `plan` and `apply` + +Validate the terraform files with - + +```shell +terraform plan --var-file=terraform_simple.tfvars +``` + +Run the terraform script with - + +```shell +terraform apply --var-file=terraform_simple.tfvars +``` + +This will launch the configured jobs and produce an output like below - + +```shell +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. + +Outputs: + +dataflow_job_id = "2024-06-05_00_41_11-4759981257849547781" +dataflow_job_url = "https://console.cloud.google.com/dataflow/jobs/us-central1/2024-06-05_00_41_11-4759981257849547781" +``` + +### Cleanup + +Once the jobs have finished running, you can cleanup by running - + +```shell +terraform destroy +``` diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf new file mode 100644 index 0000000000..1d374b551e --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/main.tf @@ -0,0 +1,54 @@ +# Add roles to the service account that will run Dataflow for validation +resource "google_project_iam_member" "validation_roles" { + for_each = var.add_policies_to_service_account ? toset([ + "roles/viewer", + "roles/storage.objectViewer", + "roles/dataflow.worker", + "roles/dataflow.admin", + "roles/spanner.databaseReader", + "roles/bigquery.dataEditor", + "roles/bigquery.jobUser", + "roles/monitoring.metricWriter", + "roles/cloudprofiler.agent" + ]) : toset([]) + project = data.google_project.project.id + role = each.key + member = var.service_account_email != null ? "serviceAccount:${var.service_account_email}" : "serviceAccount:${data.google_compute_default_service_account.gce_account.email}" +} + +resource "google_dataflow_flex_template_job" "generated" { + depends_on = [ + google_project_service.enabled_apis + ] + provider = google-beta + container_spec_gcs_path = "gs://dataflow-templates-${var.region}/latest/flex/GCS_Spanner_Data_Validator" + + parameters = { + gcsInputDirectory = var.gcs_input_directory + projectId = var.spanner_project_id + spannerHost = var.spanner_host + instanceId = var.instance_id + databaseId = var.database_id + spannerPriority = var.spanner_priority + sessionFilePath = var.session_file_path + schemaOverridesFilePath = var.schema_overrides_file_path + tableOverrides = var.table_overrides + columnOverrides = var.column_overrides + bigQueryDataset = var.bigquery_dataset + runId = var.run_id + } + + service_account_email = var.service_account_email + additional_experiments = var.additional_experiments + additional_pipeline_options = var.additional_pipeline_options + launcher_machine_type = var.launcher_machine_type + machine_type = var.machine_type + max_workers = var.max_workers + name = var.job_name + ip_configuration = var.ip_configuration + network = var.network != null ? var.host_project != null ? "projects/${var.host_project}/global/networks/${var.network}" : "projects/${var.project}/global/networks/${var.network}" : null + subnetwork = var.subnetwork != null ? var.host_project != null ? "https://www.googleapis.com/compute/v1/projects/${var.host_project}/regions/${var.region}/subnetworks/${var.subnetwork}" : "https://www.googleapis.com/compute/v1/projects/${var.project}/regions/${var.region}/subnetworks/${var.subnetwork}" : null + num_workers = var.num_workers + project = var.project + region = var.region +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf new file mode 100644 index 0000000000..57b295d112 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/outputs.tf @@ -0,0 +1,9 @@ +output "dataflow_job_id" { + value = google_dataflow_flex_template_job.generated.job_id + description = "Job id for the created Dataflow Flex Template job." +} + +output "dataflow_job_url" { + value = "https://console.cloud.google.com/dataflow/jobs/${var.region}/${google_dataflow_flex_template_job.generated.job_id}" + description = "URL for the created Dataflow Flex Template job." +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf new file mode 100644 index 0000000000..d4b1b24436 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tf @@ -0,0 +1,42 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + } + required_version = "~> 1.2" +} + +provider "google" { + project = var.project + region = var.region +} + +provider "google-beta" { + project = var.project + region = var.region +} + +# Enable the APIs +resource "google_project_service" "enabled_apis" { + for_each = toset([ + "iam.googleapis.com", + "dataflow.googleapis.com", + "storage.googleapis.com", + "spanner.googleapis.com", + "bigquery.googleapis.com" + ]) + service = each.key + project = var.project + disable_on_destroy = false +} + +# To fetch project number +data "google_project" "project" { +} + +# Fetch the default service account for Compute Engine (used by Dataflow) +data "google_compute_default_service_account" "gce_account" { + depends_on = [google_project_service.enabled_apis] +} diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tfvars b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tfvars new file mode 100644 index 0000000000..6809cd1b50 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform.tfvars @@ -0,0 +1,25 @@ +job_name = "validation-job" +project = "my-project-id" +region = "us-central1" +gcs_input_directory = "gs://my-bucket/records-path" +instance_id = "my-spanner-instance" +database_id = "my-spanner-database" +spanner_project_id = "my-spanner-project-id" +bigquery_dataset = "my_validation_dataset" + +# Optional parameters +# spanner_host = "https://batch-spanner.googleapis.com" +# spanner_priority = "HIGH" +# session_file_path = "gs://my-bucket/smt-session.json" +# schema_overrides_file_path = "gs://my-bucket/overrides.json" +# table_overrides = "[{SourceTableName1, SpannerTableName1}]" +# column_overrides = "[{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}]" +# run_id = "validation-run-001" + +# Network settings +# network = "default" +# subnetwork = "regions/us-central1/subnetworks/default" +# service_account_email = "my-service-account@my-project-id.iam.gserviceaccount.com" +# machine_type = "n1-standard-4" +# max_workers = 10 +# ip_configuration = "WORKER_IP_PRIVATE" diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars new file mode 100644 index 0000000000..ad5b944318 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/terraform_simple.tfvars @@ -0,0 +1,9 @@ +# Simplified configuration with only most common settings. +job_name = "validation-job" +project = "my-project-id" +region = "us-central1" +gcs_input_directory = "gs://my-bucket/records-path" +instance_id = "my-spanner-instance" +database_id = "my-spanner-database" +spanner_project_id = "my-spanner-project-id" +bigquery_dataset = "my_validation_dataset" diff --git a/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf new file mode 100644 index 0000000000..6e58f77b64 --- /dev/null +++ b/v2/gcs-spanner-dv/terraform/samples/gcs-spanner-dv/variables.tf @@ -0,0 +1,156 @@ +variable "add_policies_to_service_account" { + type = bool + description = "Terraform will add the required permission to the dataflow service account." + default = true +} + +variable "job_name" { + type = string + description = "Dataflow job name." +} + +variable "project" { + type = string + description = "Google Cloud Project ID where Dataflow will run." +} + +variable "host_project" { + type = string + description = "Project id hosting the network in case of a shared vpc setup." + default = null +} + +variable "region" { + type = string + description = "Google Cloud region to run Dataflow in." +} + +variable "gcs_input_directory" { + type = string + description = "This directory is used to read the AVRO files of the records read from source. For example, `gs://your-bucket/your-path`" + default = null +} + +variable "spanner_project_id" { + type = string + description = "Google Cloud Project ID (for Spanner). This is the name of the Cloud Spanner project." + default = null +} + +variable "spanner_host" { + type = string + description = "The Cloud Spanner endpoint to call in the template. For example, `https://batch-spanner.googleapis.com`." + default = null +} + +variable "instance_id" { + type = string + description = "The destination Cloud Spanner instance." +} + +variable "database_id" { + type = string + description = "The destination Cloud Spanner database." +} + +variable "spanner_priority" { + type = string + description = "The request priority for Cloud Spanner calls. The value must be one of: [`HIGH`,`MEDIUM`,`LOW`]. Defaults to `HIGH`." + default = null +} + +variable "session_file_path" { + type = string + description = "Session file path in Cloud Storage that contains mapping information from Spanner Migration Tool. Defaults to empty." + default = null +} + +variable "schema_overrides_file_path" { + type = string + description = "A file which specifies the table and the column name overrides from source to spanner. Defaults to empty." + default = null +} + +variable "table_overrides" { + type = string + description = "These are the table name overrides from source to spanner. They are written in the following format: [{SourceTableName1, SpannerTableName1}, {SourceTableName2, SpannerTableName2}]. Defaults to empty." + default = null +} + +variable "column_overrides" { + type = string + description = "These are the column name overrides from source to spanner. They are written in the following format: [{SourceTableName1.SourceColumnName1, SourceTableName1.SpannerColumnName1}, {SourceTableName2.SourceColumnName1, SourceTableName2.SpannerColumnName1}]. Defaults to empty." + default = null +} + +variable "bigquery_dataset" { + type = string + description = "The BigQuery dataset ID where the validation results will be stored. For example, `validation_report_dataset`" +} + +variable "run_id" { + type = string + description = "A unique identifier for the validation run. If not provided, the Dataflow Job Name will be used. For example, `run_20230101_120000`" + default = null +} + +# Dataflow runtime parameters +variable "additional_experiments" { + type = list(string) + description = "Additional Dataflow experiments." + default = null +} + +variable "network" { + type = string + description = "Network for Dataflow workers." + default = null +} + +variable "subnetwork" { + type = string + description = "Subnetwork for Dataflow workers." + default = null +} + +variable "service_account_email" { + type = string + description = "Service account email for Dataflow workers." + default = null +} + +variable "launcher_machine_type" { + type = string + description = "Machine type for the Dataflow launcher VM." + default = null +} + +variable "machine_type" { + type = string + description = "Machine type for Dataflow worker VMs." + default = null +} + +variable "max_workers" { + type = number + description = "Maximum number of Dataflow worker VMs." + default = null +} + +variable "ip_configuration" { + type = string + description = "IP configuration for Dataflow workers (e.g. 'WORKER_IP_PRIVATE')." + default = null +} + +variable "num_workers" { + type = number + description = "Initial number of Dataflow worker VMs." + default = null +} + +variable "additional_pipeline_options" { + type = list(string) + description = "Additional Dataflow pipeline options." + default = [] +}