tests/docker-compose.yml: move off bitnami kafka image (#2412)

* tests/docker-compose.yml: move off bitnami kafka image

Bitnami images will be gone shortly. This lets us also drop the zookeeper service since newer kafka works in KRaft mode.

* Fix kafka volume mapped data dir

* remove unneeded environment variable for kafka

* Try using 3.9.1

* try 4.1.0

* Steal kafka configuration from apm-nodejs

Author: xrmx

tests/docker-compose.yml

@@ -164,25 +164,23 @@ services:
       - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"
 
-  zookeeper:
-    image: docker.io/bitnami/zookeeper:3.8
-    ports:
-      - "2181:2181"
-    volumes:
-      - "zookeeper_data:/bitnami"
-    environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
   kafka:
-    image: docker.io/bitnami/kafka:3.1
+    image: apache/kafka:4.1.0
     ports:
       - "9092:9092"
-    volumes:
-      - "kafka_data:/bitnami"
     environment:
-      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
-      - ALLOW_PLAINTEXT_LISTENER=yes
-    depends_on:
-      - zookeeper
+      - "KAFKA_NODE_ID=1"
+      - "KAFKA_PROCESS_ROLES=broker,controller"
+      - "KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
+      - "KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
+      - "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+      - "KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093"
+      - "KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER"
+      - "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1"
+      - "KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1"
+      - "KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1"
+    volumes:
+      - "kafka_data:/var/lib/kafka/data"
 
   run_tests:
     image: ${REGISTRY:-elasticobservability}/${IMAGE_NAME:-apm-agent-python-testing}:${PYTHON_VERSION}
@@ -223,8 +221,6 @@ volumes:
     driver: local
   mysqldata:
     driver: local
-  zookeeper_data:
-    driver: local
   kafka_data:
     driver: local
   localstack_data: