diff --git a/sdks/python/apache_beam/yaml/examples/transforms/io/kafka.yaml b/sdks/python/apache_beam/yaml/examples/transforms/io/kafka.yaml index 7ab7b5ab0af0..9f2e56a3c505 100644 --- a/sdks/python/apache_beam/yaml/examples/transforms/io/kafka.yaml +++ b/sdks/python/apache_beam/yaml/examples/transforms/io/kafka.yaml @@ -93,4 +93,4 @@ pipeline: # Row(text='Fool\tThou shouldst not have been old till thou hadst') # Row(text='\tbeen wise.') # Row(text='KING LEAR\tNothing will come of nothing: speak again.') -# Row(text='\tNever, never, never, never, never!') +# Row(text='\tNever, never, never, never, never!') \ No newline at end of file diff --git a/sdks/python/apache_beam/yaml/examples/transforms/io/kafkaToKafka.yaml b/sdks/python/apache_beam/yaml/examples/transforms/io/kafkaToKafka.yaml new file mode 100644 index 000000000000..15b2fd07fbcf --- /dev/null +++ b/sdks/python/apache_beam/yaml/examples/transforms/io/kafkaToKafka.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: +# This Apache Beam YAML pipeline demonstrates a basic Kafka-to-Kafka mirroring example. +# It reads raw byte messages from a source Kafka topic and writes them to a target Kafka topic. +# No transformation or decoding is applied. +# Useful for validating Kafka I/O setup using Beam YAML DSL. + + +pipeline: + type: chain + transforms: + - type: ReadFromKafka + name: KafkaSource + config: + topic: "input-topic" + bootstrap_servers: "localhost:9092" + format: RAW + consumer_config: + group.id: beam-mirror-001 + auto.offset.reset: earliest + + - type: WriteToKafka + name: MirrorToNTopic + config: + topic: "output-topic" + bootstrap_servers: "localhost:9092" + format: RAW + producer_config_update: + key.serializer: org.apache.kafka.common.serialization.ByteArraySerializer + value.serializer: org.apache.kafka.common.serialization.ByteArraySerializer \ No newline at end of file