From f0452b1fa868e498630b7e0e2e6ccbdebdbc15b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:46:56 +0000 Subject: [PATCH 1/5] Initial plan From 5ba83723945044b5885b0e690908f1e1dd9347e3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:48:43 +0000 Subject: [PATCH 2/5] Add enhanced Kafka integration tests for consumer groups, admin operations, and topic deletion Co-authored-by: phlax <454682+phlax@users.noreply.github.com> --- kafka/example.rst | 78 +++++++++++++++++++++++++++++++++++++++++++++++ kafka/verify.sh | 36 ++++++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/kafka/example.rst b/kafka/example.rst index 22c4ac0e..2661392b 100644 --- a/kafka/example.rst +++ b/kafka/example.rst @@ -125,6 +125,84 @@ Envoy also records cluster stats for the Kafka service: cluster.kafka_service.membership_healthy: 1 cluster.kafka_service.membership_total: 1 + +Step 8: Test consumer groups +**************************** + +Consumer groups allow multiple consumers to coordinate consumption of a topic. +When consumers join a group, they use the Kafka group coordination protocol +which Envoy proxies transparently. + +Start a consumer in a group. It will wait for messages and then exit after +a timeout: + +.. code-block:: console + + $ docker compose run --rm kafka-client \ + kafka-console-consumer --bootstrap-server proxy:10000 \ + --topic $TOPIC --group test-group --timeout-ms 5000 + +The consumer group protocol generates additional request types. Check that +the group coordination metrics have incremented: + +.. code-block:: console + + $ curl -s "http://localhost:8001/stats?filter=kafka.kafka_broker" | grep -E "(find_coordinator|join_group|sync_group|leave_group)" | grep -v ": 0" + kafka.kafka_broker.request.find_coordinator_request: 1 + kafka.kafka_broker.request.join_group_request: 1 + kafka.kafka_broker.request.leave_group_request: 1 + kafka.kafka_broker.response.find_coordinator_response: 1 + kafka.kafka_broker.response.join_group_response: 1 + kafka.kafka_broker.response.leave_group_response: 1 + + +Step 9: Test additional admin operations +**************************************** + +The Kafka admin client supports various topic management operations. Test +altering topic configuration: + +.. code-block:: console + + $ docker compose run --rm kafka-client \ + kafka-configs --bootstrap-server proxy:10000 \ + --alter --entity-type topics --entity-name $TOPIC \ + --add-config retention.ms=86400000 + +Add partitions to the topic: + +.. code-block:: console + + $ docker compose run --rm kafka-client \ + kafka-topics --bootstrap-server proxy:10000 \ + --alter --topic $TOPIC --partitions 3 + +Check the admin operation metrics: + +.. code-block:: console + + $ curl -s "http://localhost:8001/stats?filter=kafka.kafka_broker" | grep -E "(alter_configs|create_partitions)" | grep -v ": 0" + kafka.kafka_broker.request.alter_configs_request: 1 + kafka.kafka_broker.request.create_partitions_request: 1 + + +Step 10: Delete the topic +************************* + +Clean up by deleting the test topic: + +.. code-block:: console + + $ docker compose run --rm kafka-client \ + kafka-topics --bootstrap-server proxy:10000 --delete --topic $TOPIC + +Verify the delete operation was tracked: + +.. code-block:: console + + $ curl -s "http://localhost:8001/stats?filter=kafka.kafka_broker.request.delete_topics" | grep -v ": 0" + kafka.kafka_broker.request.delete_topics_request: 1 + .. seealso:: :ref:`Envoy Kafka broker filter ` diff --git a/kafka/verify.sh b/kafka/verify.sh index 24f7b769..c47b1284 100755 --- a/kafka/verify.sh +++ b/kafka/verify.sh @@ -79,3 +79,39 @@ for stat in "${EXPECTED_BROKER_STATS[@]}"; do "$stat" \ "http://localhost:${PORT_ADMIN}/stats?filter=${filter}" done + +run_log "Test consumer group coordination" +# Run a consumer in a group - it will timeout after 5s +kafka_client kafka-console-consumer --bootstrap-server proxy:10000 \ + --topic $TOPIC --group test-group --timeout-ms 5000 || true + +run_log "Check consumer group metrics" +EXPECTED_GROUP_STATS=( + "kafka.kafka_broker.request.find_coordinator_request" + "kafka.kafka_broker.request.join_group_request" + "kafka.kafka_broker.response.find_coordinator_response" + "kafka.kafka_broker.response.join_group_response") +for stat in "${EXPECTED_GROUP_STATS[@]}"; do + has_metric_with_at_least_1 "${stat}" +done + +run_log "Test alter topic config" +kafka_client kafka-configs --bootstrap-server proxy:10000 \ + --alter --entity-type topics --entity-name $TOPIC \ + --add-config retention.ms=86400000 + +run_log "Check alter_configs metric" +has_metric_with_at_least_1 "kafka.kafka_broker.request.alter_configs_request" + +run_log "Test add partitions" +kafka_client kafka-topics --bootstrap-server proxy:10000 \ + --alter --topic $TOPIC --partitions 3 + +run_log "Check create_partitions metric" +has_metric_with_at_least_1 "kafka.kafka_broker.request.create_partitions_request" + +run_log "Test delete topic" +kafka_client kafka-topics --bootstrap-server proxy:10000 --delete --topic $TOPIC + +run_log "Check delete_topics metric" +has_metric_with_at_least_1 "kafka.kafka_broker.request.delete_topics_request" From c7952eaa5d84000173fb12e987482eeb66c5fe41 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:51:41 +0000 Subject: [PATCH 3/5] Fix metric name from alter_configs to incremental_alter_configs Co-authored-by: phlax <454682+phlax@users.noreply.github.com> --- kafka/example.rst | 4 ++-- kafka/verify.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kafka/example.rst b/kafka/example.rst index 2661392b..52d0b06b 100644 --- a/kafka/example.rst +++ b/kafka/example.rst @@ -181,8 +181,8 @@ Check the admin operation metrics: .. code-block:: console - $ curl -s "http://localhost:8001/stats?filter=kafka.kafka_broker" | grep -E "(alter_configs|create_partitions)" | grep -v ": 0" - kafka.kafka_broker.request.alter_configs_request: 1 + $ curl -s "http://localhost:8001/stats?filter=kafka.kafka_broker" | grep -E "(incremental_alter_configs|create_partitions)" | grep -v ": 0" + kafka.kafka_broker.request.incremental_alter_configs_request: 1 kafka.kafka_broker.request.create_partitions_request: 1 diff --git a/kafka/verify.sh b/kafka/verify.sh index c47b1284..06d3079e 100755 --- a/kafka/verify.sh +++ b/kafka/verify.sh @@ -100,8 +100,8 @@ kafka_client kafka-configs --bootstrap-server proxy:10000 \ --alter --entity-type topics --entity-name $TOPIC \ --add-config retention.ms=86400000 -run_log "Check alter_configs metric" -has_metric_with_at_least_1 "kafka.kafka_broker.request.alter_configs_request" +run_log "Check incremental_alter_configs metric" +has_metric_with_at_least_1 "kafka.kafka_broker.request.incremental_alter_configs_request" run_log "Test add partitions" kafka_client kafka-topics --bootstrap-server proxy:10000 \ From 0f8f894c89ba9f9c2790885b8257a44022ef9afb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:53:34 +0000 Subject: [PATCH 4/5] Remove trailing whitespace in example.rst Co-authored-by: phlax <454682+phlax@users.noreply.github.com> --- kafka/example.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/example.rst b/kafka/example.rst index 52d0b06b..4684e21f 100644 --- a/kafka/example.rst +++ b/kafka/example.rst @@ -133,7 +133,7 @@ Consumer groups allow multiple consumers to coordinate consumption of a topic. When consumers join a group, they use the Kafka group coordination protocol which Envoy proxies transparently. -Start a consumer in a group. It will wait for messages and then exit after +Start a consumer in a group. It will wait for messages and then exit after a timeout: .. code-block:: console From b09a1cbfb8834befd2eb76acdc2906b383f5be4b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:54:07 +0000 Subject: [PATCH 5/5] Add comment explaining why timeout exit code is ignored Co-authored-by: phlax <454682+phlax@users.noreply.github.com> --- kafka/verify.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kafka/verify.sh b/kafka/verify.sh index 06d3079e..4d8a2ea7 100755 --- a/kafka/verify.sh +++ b/kafka/verify.sh @@ -82,6 +82,7 @@ done run_log "Test consumer group coordination" # Run a consumer in a group - it will timeout after 5s +# The timeout is expected since there are no new messages, so we ignore the exit code kafka_client kafka-console-consumer --bootstrap-server proxy:10000 \ --topic $TOPIC --group test-group --timeout-ms 5000 || true