From 2e2a39a8fcbaaa350625325cfad5080c6d87c173 Mon Sep 17 00:00:00 2001 From: Paulo Vital Date: Wed, 20 Aug 2025 14:12:57 +0200 Subject: [PATCH 1/2] ci: Update Kafka container. Moving to use the `ubuntu/kafka` container image as it has support to `amd64(x86_64)`, `arm64`, `ppc64le`, and `s390x`. Signed-off-by: Paulo Vital --- .circleci/config.yml | 22 ++++++++++------------ .tekton/task.yaml | 25 ++++++++++--------------- docker-compose.yml | 20 +++++++++++--------- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fef25d9..d4451306 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -244,18 +244,16 @@ jobs: - store-pytest-results - store-coverage-report - py312kafka: + py313kafka: docker: - - image: public.ecr.aws/docker/library/python:3.12 - - image: public.ecr.aws/bitnami/kafka:3.9.0 + - image: public.ecr.aws/docker/library/python:3.13 + - image: public.ecr.aws/ubuntu/zookeeper:latest + environment: + TZ: UTC + - image: public.ecr.aws/ubuntu/kafka:latest environment: - KAFKA_CFG_NODE_ID: 0 - KAFKA_CFG_PROCESS_ROLES: controller,broker - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093 - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094 + TZ: UTC + ZOOKEEPER_HOST: localhost working_directory: ~/repo steps: - checkout @@ -309,7 +307,7 @@ workflows: - py39cassandra - py39gevent - py312aws - - py312kafka + - py313kafka - autowrapt: matrix: parameters: @@ -322,5 +320,5 @@ workflows: - py39cassandra - py39gevent - py312aws - - py312kafka + - py313kafka - autowrapt diff --git a/.tekton/task.yaml b/.tekton/task.yaml index e5d79c92..3f3d98ec 100644 --- a/.tekton/task.yaml +++ b/.tekton/task.yaml @@ -166,23 +166,18 @@ metadata: name: python-tracer-unittest-kafka-task spec: sidecars: + - name: zookeeper + image: public.ecr.aws/ubuntu/zookeeper:latest + env: + - name: TZ + value: "UTC" - name: kafka - image: public.ecr.aws/bitnami/kafka:3.9.0 + image: public.ecr.aws/ubuntu/kafka:latest env: - - name: KAFKA_CFG_NODE_ID - value: "0" - - name: KAFKA_CFG_PROCESS_ROLES - value: "controller,broker" - - name: KAFKA_CFG_LISTENERS - value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" - - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP - value: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT" - - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS - value: "0@kafka:9093" - - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES - value: "CONTROLLER" - - name: KAFKA_CFG_ADVERTISED_LISTENERS - value: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094" + - name: TZ + value: "UTC" + - name: ZOOKEEPER_HOST + value: zookeeper params: - name: imageDigest type: string diff --git a/docker-compose.yml b/docker-compose.yml index 45393b76..35dea903 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,16 +61,18 @@ services: - "8681:8681" - "8682:8682" + # Sidecar container for Kafka + zookeeper: + image: public.ecr.aws/ubuntu/zookeeper:latest + ports: + - 2181:2181 + environment: + - TZ=UTC + kafka: - image: public.ecr.aws/bitnami/kafka:latest + image: public.ecr.aws/ubuntu/kafka:latest # works on amd64, arm64, ppc64le and s390x ports: - '9092:9092' - - '9094:9094' environment: - - KAFKA_CFG_NODE_ID=0 - - KAFKA_CFG_PROCESS_ROLES=controller,broker - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT - - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 - - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094 + - TZ=UTC + - ZOOKEEPER_HOST=zookeeper From a02ff5883d5fdb87a1347bb19f78fa8713303779 Mon Sep 17 00:00:00 2001 From: Cagri Yonca Date: Thu, 2 Oct 2025 15:32:25 +0200 Subject: [PATCH 2/2] ci: Fix Kafka connection issues to ubuntu/kafka Signed-off-by: Cagri Yonca --- .circleci/config.yml | 25 +++++++++++++++++++++++-- .tekton/task.yaml | 34 +++++++++++++++++++++++++++++++--- docker-compose.yml | 37 +++++++++++++++++++++++++++++-------- 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4451306..1bba6674 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -247,13 +247,34 @@ jobs: py313kafka: docker: - image: public.ecr.aws/docker/library/python:3.13 - - image: public.ecr.aws/ubuntu/zookeeper:latest + - image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge environment: TZ: UTC - - image: public.ecr.aws/ubuntu/kafka:latest + - image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge environment: TZ: UTC ZOOKEEPER_HOST: localhost + ZOOKEEPER_PORT: 2181 + command: + - /opt/kafka/config/server.properties + - --override + - listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094 + - --override + - advertised.listeners=INTERNAL://localhost:9093,EXTERNAL://localhost:9094 + - --override + - listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + - --override + - inter.broker.listener.name=INTERNAL + - --override + - broker.id=1 + - --override + - offsets.topic.replication.factor=1 + - --override + - transaction.state.log.replication.factor=1 + - --override + - transaction.state.log.min.isr=1 + - --override + - auto.create.topics.enable=true working_directory: ~/repo steps: - checkout diff --git a/.tekton/task.yaml b/.tekton/task.yaml index 3f3d98ec..0a9a6d05 100644 --- a/.tekton/task.yaml +++ b/.tekton/task.yaml @@ -167,17 +167,45 @@ metadata: spec: sidecars: - name: zookeeper - image: public.ecr.aws/ubuntu/zookeeper:latest + image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge + ports: + - containerPort: 9093 env: - name: TZ value: "UTC" - name: kafka - image: public.ecr.aws/ubuntu/kafka:latest + image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge env: - name: TZ value: "UTC" - name: ZOOKEEPER_HOST - value: zookeeper + value: localhost + - name: ZOOKEEPER_PORT + value: "2181" + ports: + - containerPort: 9093 + - containerPort: 9094 + command: + - /opt/kafka/bin/kafka-server-start.sh + - /opt/kafka/config/server.properties + - --override + - listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094 + - --override + - advertised.listeners=INTERNAL://localhost:9093,EXTERNAL://localhost:9094 + - --override + - listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + - --override + - inter.broker.listener.name=INTERNAL + - --override + - broker.id=1 + - --override + - offsets.topic.replication.factor=1 + - --override + - transaction.state.log.replication.factor=1 + - --override + - transaction.state.log.min.isr=1 + - --override + - auto.create.topics.enable=true params: - name: imageDigest type: string diff --git a/docker-compose.yml b/docker-compose.yml index 35dea903..299806a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,16 +63,37 @@ services: # Sidecar container for Kafka zookeeper: - image: public.ecr.aws/ubuntu/zookeeper:latest - ports: - - 2181:2181 - environment: - - TZ=UTC + image: public.ecr.aws/ubuntu/zookeeper:3.1-22.04_edge + ports: ["2181:2181"] + environment: [ "TZ=UTC" ] kafka: - image: public.ecr.aws/ubuntu/kafka:latest # works on amd64, arm64, ppc64le and s390x - ports: - - '9092:9092' + image: public.ecr.aws/ubuntu/kafka:3.1-22.04_edge + depends_on: [zookeeper] + ports: + - "9094:9094" + - "9093:9093" environment: - TZ=UTC - ZOOKEEPER_HOST=zookeeper + - ZOOKEEPER_PORT=2181 + command: + - /opt/kafka/config/server.properties + - --override + - listeners=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9094 + - --override + - advertised.listeners=INTERNAL://kafka:9093,EXTERNAL://127.0.0.1:9094 + - --override + - listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + - --override + - inter.broker.listener.name=INTERNAL + - --override + - broker.id=1 + - --override + - offsets.topic.replication.factor=1 + - --override + - transaction.state.log.replication.factor=1 + - --override + - transaction.state.log.min.isr=1 + - --override + - auto.create.topics.enable=true