From 6175228c0cd572d20d64678d3a71121cbd704d77 Mon Sep 17 00:00:00 2001 From: Ivan Lam Date: Mon, 29 Jun 2026 17:40:28 +0000 Subject: [PATCH 1/4] fix: enable streaming pull in Pub/Sub source connector configuration to match with default configs in Google Cloud Console UI --- .../main.tf | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/managedkafka/managedkafka_create_connector_pubsub_source/main.tf b/managedkafka/managedkafka_create_connector_pubsub_source/main.tf index 43cc77ce..4ec7fe33 100644 --- a/managedkafka/managedkafka_create_connector_pubsub_source/main.tf +++ b/managedkafka/managedkafka_create_connector_pubsub_source/main.tf @@ -79,14 +79,15 @@ resource "google_managed_kafka_connector" "example-pubsub-source-connector" { location = "us-central1" configs = { - "connector.class" = "com.google.pubsub.kafka.source.CloudPubSubSourceConnector" - "name" = "my-pubsub-source-connector" - "tasks.max" = "3" - "kafka.topic" = "GMK_TOPIC_ID" - "cps.subscription" = "CPS_SUBSCRIPTION_ID" - "cps.project" = data.google_project.default.project_id - "value.converter" = "org.apache.kafka.connect.converters.ByteArrayConverter" - "key.converter" = "org.apache.kafka.connect.storage.StringConverter" + "connector.class" = "com.google.pubsub.kafka.source.CloudPubSubSourceConnector" + "name" = "my-pubsub-source-connector" + "tasks.max" = "3" + "kafka.topic" = "GMK_TOPIC_ID" + "cps.subscription" = "CPS_SUBSCRIPTION_ID" + "cps.project" = data.google_project.default.project_id + "cps.streamingPull.enable" = "true" + "value.converter" = "org.apache.kafka.connect.converters.ByteArrayConverter" + "key.converter" = "org.apache.kafka.connect.storage.StringConverter" } provider = google-beta From ea8f8f82d0460825ad420caa7f666536d74aeb25 Mon Sep 17 00:00:00 2001 From: Ivan Lam Date: Mon, 29 Jun 2026 17:44:07 +0000 Subject: [PATCH 2/4] fix: update Kafka connector converters from StringConverter to ByteArrayConverter to match with config in Google Console UI --- .../managedkafka_create_connector_pubsub_sink/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managedkafka/managedkafka_create_connector_pubsub_sink/main.tf b/managedkafka/managedkafka_create_connector_pubsub_sink/main.tf index f0622df7..5402dab6 100644 --- a/managedkafka/managedkafka_create_connector_pubsub_sink/main.tf +++ b/managedkafka/managedkafka_create_connector_pubsub_sink/main.tf @@ -85,8 +85,8 @@ resource "google_managed_kafka_connector" "example-pubsub-sink-connector" { "topics" = "TOPIC_NAME" "cps.topic" = "CPS_TOPIC_NAME" "cps.project" = "CPS_PROJECT_NAME" - "value.converter" = "org.apache.kafka.connect.storage.StringConverter" - "key.converter" = "org.apache.kafka.connect.storage.StringConverter" + "value.converter" = "org.apache.kafka.connect.converters.ByteArrayConverter" + "key.converter" = "org.apache.kafka.connect.converters.ByteArrayConverter" } provider = google-beta From bdd284a54976f9746e51bbfb8467e9ad83686d8a Mon Sep 17 00:00:00 2001 From: Ivan Lam Date: Tue, 30 Jun 2026 15:16:55 +0000 Subject: [PATCH 3/4] fix: update MirrorMaker 2 bootstrap server addresses to use dynamic Managed Kafka cluster endpoints --- .../managedkafka_create_connector_mirrormaker2_source/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf b/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf index 0489ef59..8f12cef0 100644 --- a/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf +++ b/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf @@ -118,8 +118,8 @@ resource "google_managed_kafka_connector" "default" { "topics" = ".*" # Replicate all topics from the source # The value for bootstrap.servers is a comma-separated list of hostname:port pairs # for one or more Kafka brokers in the source/target cluster. - "source.cluster.bootstrap.servers" = "source_cluster_dns" - "target.cluster.bootstrap.servers" = "target_cluster_dns" + "source.cluster.bootstrap.servers" = "bootstrap.${google_managed_kafka_cluster.source.cluster_id}.${google_managed_kafka_cluster.source.location}.managedkafka.${data.google_project.default.project_id}.cloud.goog:9092" + "target.cluster.bootstrap.servers" = "bootstrap.${google_managed_kafka_cluster.target.cluster_id}.${google_managed_kafka_cluster.target.location}.managedkafka.${data.google_project.default.project_id}.cloud.goog:9092" # You can define an exclusion policy for topics as follows: # To exclude internal MirrorMaker 2 topics, internal topics and replicated topics,. "topics.exclude" = "mm2.*\\.internal,.*\\.replica,__.*" From 686b7704f8463e8098daf46c093e8ff02bcad540 Mon Sep 17 00:00:00 2001 From: Ivan Lam Date: Tue, 30 Jun 2026 16:02:05 +0000 Subject: [PATCH 4/4] fix: update dns_domain_names to use dynamic cluster and project references for MirrorMaker2 connector --- .../managedkafka_create_connector_mirrormaker2_source/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf b/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf index 8f12cef0..5d08ba0f 100644 --- a/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf +++ b/managedkafka/managedkafka_create_connector_mirrormaker2_source/main.tf @@ -76,7 +76,7 @@ resource "google_managed_kafka_connect_cluster" "default" { # "GMK_CLUSTER_ID.REGION.managedkafka.PROJECT_ID.cloud.goog.*" # Please note that we do NOT need to add the DNS name of the primary Kafka cluster to the # `dns_domain_names` list, as our Connect cluster configures that automatically. - dns_domain_names = ["DNS_DOMAIN_NAME"] + dns_domain_names = ["${google_managed_kafka_cluster.source.cluster_id}.${google_managed_kafka_cluster.source.location}.managedkafka.${data.google_project.default.project_id}.cloud.goog."] } } }