From fbb3c286147d3dec1f7fad1b7cd17d1c2a764edd Mon Sep 17 00:00:00 2001 From: Sonam Choubey Date: Wed, 15 Jul 2026 17:05:13 +0530 Subject: [PATCH 1/4] DBR-3198-updat-docker-compose --- README.md | 32 ++++++- cluster-example/README.md | 5 ++ cluster-example/docker-compose.yml | 140 ++++++++++++++--------------- local-setup/README.md | 6 ++ local-setup/docker-compose.yml | 66 ++++++++------ 5 files changed, 142 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 862710b..3a4f2a1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,35 @@ # Mendix Business Events Module -## Table of contents +## Overview -### cluster-example +This repository provides Docker Compose configurations to help you quickly set up Apache Kafka for use with Mendix Business Events. Both setups now use Apache Kafka 3.8.0 with KRaft mode, eliminating the need for Zookeeper and providing built-in Change Data Capture (CDC) capabilities. -Provides and example of a Apache Zookeeper cluster with an Apache Kafka cluster +## What's Inside ### local-setup -Creates a local running Docker instance of Zookeeper and Kafka in order to test local modelling capabilities +A simple single-node Kafka setup designed for local development and testing. Perfect for getting started with Mendix Business Events on your local machine. + +### cluster-example + +A 3-node Kafka cluster configuration that demonstrates a more production-like setup with high availability and replication. + +## Key Features + +### Modern Kafka Architecture +- **Apache Kafka 3.8.0** with KRaft mode (no Zookeeper required) +- Simplified deployment and management +- Improved performance and scalability + +### Change Data Capture (CDC) +Both setups include Kafka Connect with Debezium, giving you powerful CDC capabilities out of the box: +- Capture real-time database changes from PostgreSQL, MySQL, MongoDB, SQL Server, Oracle, and more +- Stream database events directly to Kafka topics +- Build event-driven architectures based on data modifications +- Enable real-time data synchronization across systems + +CDC is ideal for scenarios like: +- Keeping multiple systems in sync +- Building event sourcing patterns +- Creating materialized views +- Integrating microservices with eventual consistency diff --git a/cluster-example/README.md b/cluster-example/README.md index b62688c..40b4343 100644 --- a/cluster-example/README.md +++ b/cluster-example/README.md @@ -1,5 +1,10 @@ # Business events cluster example setup +#whats new +This setup now uses **Apache Kafka 3.8.0** with **KRaft mode**, which means: +- No Zookeeper required - simpler architecture and easier management +- Built-in CDC support through Kafka Connect and Debezium + ## Purpose The examples provided in this setup is purely for illustration and is not intended to be a reference architecture for deployment of Apache Zookeeper and/or Apache Kafka. diff --git a/cluster-example/docker-compose.yml b/cluster-example/docker-compose.yml index db37859..a3e3eba 100644 --- a/cluster-example/docker-compose.yml +++ b/cluster-example/docker-compose.yml @@ -1,118 +1,110 @@ --- -version: "3.0" - services: - zookeeper-1: - container_name: zookeeper-1 - hostname: zookeeper-1 - image: confluentinc/cp-zookeeper:6.2.6 - environment: - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_CLIENT_PORT: 22181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888 - network_mode: host - extra_hosts: - - "moby:127.0.0.1" - - zookeeper-2: - container_name: zookeeper-2 - hostname: zookeeper-2 - image: confluentinc/cp-zookeeper:6.2.6 - environment: - ZOOKEEPER_SERVER_ID: 2 - ZOOKEEPER_CLIENT_PORT: 32181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888 - network_mode: host - extra_hosts: - - "moby:127.0.0.1" - - zookeeper-3: - container_name: zookeeper-3 - hostname: zookeeper-3 - image: confluentinc/cp-zookeeper:6.2.6 - environment: - ZOOKEEPER_SERVER_ID: 3 - ZOOKEEPER_CLIENT_PORT: 42181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888 - network_mode: host - extra_hosts: - - "moby:127.0.0.1" - kafka-1: container_name: kafka-1 hostname: kafka-1 - image: confluentinc/cp-kafka:6.2.6 + image: apache/kafka:3.8.0 network_mode: host - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181 + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:19092,CONTROLLER://localhost:19093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 KAFKA_DEFAULT_REPLICATION_FACTOR: 2 + KAFKA_MIN_INSYNC_REPLICAS: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_LOG_RETENTION_HOURS: 120 KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_COMPRESSION_TYPE: zstd - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_LOG_DIRS: /tmp/kraft-combined-logs + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk extra_hosts: - "moby:127.0.0.1" kafka-2: container_name: kafka-2 hostname: kafka-2 - image: confluentinc/cp-kafka:6.2.6 + image: apache/kafka:3.8.0 network_mode: host - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 environment: - KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181 + KAFKA_NODE_ID: 2 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 KAFKA_DEFAULT_REPLICATION_FACTOR: 2 + KAFKA_MIN_INSYNC_REPLICAS: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_LOG_RETENTION_HOURS: 120 KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_COMPRESSION_TYPE: zstd - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_LOG_DIRS: /tmp/kraft-combined-logs + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk extra_hosts: - "moby:127.0.0.1" kafka-3: container_name: kafka-3 hostname: kafka-3 - image: confluentinc/cp-kafka:6.2.6 + image: apache/kafka:3.8.0 network_mode: host - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 environment: - KAFKA_BROKER_ID: 3 - KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181 + KAFKA_NODE_ID: 3 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:39092,CONTROLLER://localhost:39093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 KAFKA_DEFAULT_REPLICATION_FACTOR: 2 + KAFKA_MIN_INSYNC_REPLICAS: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_LOG_RETENTION_HOURS: 120 KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_COMPRESSION_TYPE: zstd - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + KAFKA_LOG_DIRS: /tmp/kraft-combined-logs + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk + extra_hosts: + - "moby:127.0.0.1" + + kafka-connect: + container_name: kafka-connect + hostname: kafka-connect + image: quay.io/debezium/connect:2.7 + network_mode: host + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + environment: + BOOTSTRAP_SERVERS: localhost:19092,localhost:29092,localhost:39092 + GROUP_ID: 1 + CONFIG_STORAGE_TOPIC: connect_configs + OFFSET_STORAGE_TOPIC: connect_offsets + STATUS_STORAGE_TOPIC: connect_statuses + CONFIG_STORAGE_REPLICATION_FACTOR: 2 + OFFSET_STORAGE_REPLICATION_FACTOR: 2 + STATUS_STORAGE_REPLICATION_FACTOR: 2 + KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter + VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false" + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" + REST_ADVERTISED_HOST_NAME: localhost + REST_PORT: 8083 extra_hosts: - "moby:127.0.0.1" diff --git a/local-setup/README.md b/local-setup/README.md index 54b0f69..c4ae1e3 100644 --- a/local-setup/README.md +++ b/local-setup/README.md @@ -3,6 +3,12 @@ This project helps to get you quickly up and running using [Mendix Business Events](https://docs.mendix.com/appstore/modules/business-events/). The main component is a Docker compose file, so you quickly have the local resources available to support using business events. +#whats new +This setup now uses **Apache Kafka 3.8.0** with **KRaft mode**, which means: +- No Zookeeper required - simpler architecture and easier management +- Built-in CDC support through Kafka Connect and Debezium + + Should you need to install Docker on your local machine, you can get downloads and instructions [here](https://docs.docker.com/engine/install/) If you are using `Microsoft's Visual Studio Code`: diff --git a/local-setup/docker-compose.yml b/local-setup/docker-compose.yml index fe8b405..f6646c4 100644 --- a/local-setup/docker-compose.yml +++ b/local-setup/docker-compose.yml @@ -1,44 +1,52 @@ --- -version: "3.0" services: - zk: - image: confluentinc/cp-zookeeper:6.2.0 - hostname: zk - container_name: zk - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - mem_limit: 200m - restart: always - kafka: - image: confluentinc/cp-kafka:6.2.0 + image: apache/kafka:3.8.0 hostname: kafka container_name: kafka ports: - "9092:9092" - "9094:9094" - depends_on: - - zk environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zk:2181 + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://kafka:19092,CONTROLLER://kafka:9093,PE://0.0.0.0:9092,LISTENER_P://0.0.0.0:9094 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:19092,PE://127.0.0.1:9092,LISTENER_P://10.211.55.2:9094 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PE:PLAINTEXT,LISTENER_P:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_COMPRESSION_TYPE: lz4 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_DELETE_TOPIC_ENABLE: "true" - KAFKA_MIN_INSYNC_REPLICAS: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PE:PLAINTEXT,LISTENER_P:PLAINTEXT - CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_NUM_PARTITIONS: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_LOG_DIRS: /tmp/kraft-combined-logs + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk mem_limit: 1500m restart: always + + kafka-connect: + image: quay.io/debezium/connect:2.7 + hostname: kafka-connect + container_name: kafka-connect + ports: + - "8083:8083" + depends_on: + - kafka + environment: + BOOTSTRAP_SERVERS: kafka:19092 + GROUP_ID: 1 + CONFIG_STORAGE_TOPIC: connect_configs + OFFSET_STORAGE_TOPIC: connect_offsets + STATUS_STORAGE_TOPIC: connect_statuses + CONFIG_STORAGE_REPLICATION_FACTOR: 1 + OFFSET_STORAGE_REPLICATION_FACTOR: 1 + STATUS_STORAGE_REPLICATION_FACTOR: 1 + KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter + VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false" + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" + mem_limit: 1000m + restart: always From 1e621c820a03989affcd639cdfa0ba6aa71501eb Mon Sep 17 00:00:00 2001 From: Sonam Choubey Date: Fri, 17 Jul 2026 14:40:33 +0530 Subject: [PATCH 2/4] DBR-3198-updat-docker-compose --- README.md | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 3a4f2a1..f54eeba 100644 --- a/README.md +++ b/README.md @@ -14,22 +14,8 @@ A simple single-node Kafka setup designed for local development and testing. Per A 3-node Kafka cluster configuration that demonstrates a more production-like setup with high availability and replication. -## Key Features - -### Modern Kafka Architecture -- **Apache Kafka 3.8.0** with KRaft mode (no Zookeeper required) -- Simplified deployment and management -- Improved performance and scalability - -### Change Data Capture (CDC) -Both setups include Kafka Connect with Debezium, giving you powerful CDC capabilities out of the box: -- Capture real-time database changes from PostgreSQL, MySQL, MongoDB, SQL Server, Oracle, and more -- Stream database events directly to Kafka topics -- Build event-driven architectures based on data modifications -- Enable real-time data synchronization across systems - -CDC is ideal for scenarios like: -- Keeping multiple systems in sync -- Building event sourcing patterns -- Creating materialized views -- Integrating microservices with eventual consistency +# whats new +This setup now uses **Apache Kafka 3.8.0** with **KRaft mode**, which means: +- No Zookeeper required - simpler architecture and easier management +- Built-in CDC support through Kafka Connect and Debezium + From 3efde2869347a0a3e8afa12bd258fd770b309420 Mon Sep 17 00:00:00 2001 From: Sonam Choubey Date: Thu, 23 Jul 2026 15:24:06 +0530 Subject: [PATCH 3/4] eliminating-code-which-is-not-needed --- local-setup/docker-compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/local-setup/docker-compose.yml b/local-setup/docker-compose.yml index f6646c4..9d16bb3 100644 --- a/local-setup/docker-compose.yml +++ b/local-setup/docker-compose.yml @@ -16,14 +16,9 @@ services: KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PE:PLAINTEXT,LISTENER_P:PLAINTEXT KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_NUM_PARTITIONS: 1 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_LOG_DIRS: /tmp/kraft-combined-logs - CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk mem_limit: 1500m restart: always From 0887ade0f542f8b6391a099209591a0060db7274 Mon Sep 17 00:00:00 2001 From: Sonam Choubey Date: Thu, 23 Jul 2026 15:35:17 +0530 Subject: [PATCH 4/4] eliminating-code-which-is-not-needed-for-cluser-example-too --- cluster-example/docker-compose.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/cluster-example/docker-compose.yml b/cluster-example/docker-compose.yml index a3e3eba..e40492e 100644 --- a/cluster-example/docker-compose.yml +++ b/cluster-example/docker-compose.yml @@ -16,15 +16,6 @@ services: KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 - KAFKA_DEFAULT_REPLICATION_FACTOR: 2 - KAFKA_MIN_INSYNC_REPLICAS: 1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_LOG_RETENTION_HOURS: 120 - KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_LOG_DIRS: /tmp/kraft-combined-logs - CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk extra_hosts: - "moby:127.0.0.1" @@ -43,15 +34,6 @@ services: KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 - KAFKA_DEFAULT_REPLICATION_FACTOR: 2 - KAFKA_MIN_INSYNC_REPLICAS: 1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_LOG_RETENTION_HOURS: 120 - KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_LOG_DIRS: /tmp/kraft-combined-logs - CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk extra_hosts: - "moby:127.0.0.1" @@ -70,15 +52,6 @@ services: KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 - KAFKA_DEFAULT_REPLICATION_FACTOR: 2 - KAFKA_MIN_INSYNC_REPLICAS: 1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_LOG_RETENTION_HOURS: 120 - KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_LOG_DIRS: /tmp/kraft-combined-logs - CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk extra_hosts: - "moby:127.0.0.1"