diff --git a/README.md b/README.md index 862710b..f54eeba 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,21 @@ # Mendix Business Events Module -## Table of contents +## Overview -### cluster-example +This repository provides Docker Compose configurations to help you quickly set up Apache Kafka for use with Mendix Business Events. Both setups now use Apache Kafka 3.8.0 with KRaft mode, eliminating the need for Zookeeper and providing built-in Change Data Capture (CDC) capabilities. -Provides and example of a Apache Zookeeper cluster with an Apache Kafka cluster +## What's Inside ### local-setup -Creates a local running Docker instance of Zookeeper and Kafka in order to test local modelling capabilities +A simple single-node Kafka setup designed for local development and testing. Perfect for getting started with Mendix Business Events on your local machine. + +### cluster-example + +A 3-node Kafka cluster configuration that demonstrates a more production-like setup with high availability and replication. + +# whats new +This setup now uses **Apache Kafka 3.8.0** with **KRaft mode**, which means: +- No Zookeeper required - simpler architecture and easier management +- Built-in CDC support through Kafka Connect and Debezium + diff --git a/cluster-example/README.md b/cluster-example/README.md index b62688c..40b4343 100644 --- a/cluster-example/README.md +++ b/cluster-example/README.md @@ -1,5 +1,10 @@ # Business events cluster example setup +#whats new +This setup now uses **Apache Kafka 3.8.0** with **KRaft mode**, which means: +- No Zookeeper required - simpler architecture and easier management +- Built-in CDC support through Kafka Connect and Debezium + ## Purpose The examples provided in this setup is purely for illustration and is not intended to be a reference architecture for deployment of Apache Zookeeper and/or Apache Kafka. diff --git a/cluster-example/docker-compose.yml b/cluster-example/docker-compose.yml index db37859..e40492e 100644 --- a/cluster-example/docker-compose.yml +++ b/cluster-example/docker-compose.yml @@ -1,118 +1,83 @@ --- -version: "3.0" - services: - zookeeper-1: - container_name: zookeeper-1 - hostname: zookeeper-1 - image: confluentinc/cp-zookeeper:6.2.6 - environment: - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_CLIENT_PORT: 22181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888 - network_mode: host - extra_hosts: - - "moby:127.0.0.1" - - zookeeper-2: - container_name: zookeeper-2 - hostname: zookeeper-2 - image: confluentinc/cp-zookeeper:6.2.6 - environment: - ZOOKEEPER_SERVER_ID: 2 - ZOOKEEPER_CLIENT_PORT: 32181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888 - network_mode: host - extra_hosts: - - "moby:127.0.0.1" - - zookeeper-3: - container_name: zookeeper-3 - hostname: zookeeper-3 - image: confluentinc/cp-zookeeper:6.2.6 - environment: - ZOOKEEPER_SERVER_ID: 3 - ZOOKEEPER_CLIENT_PORT: 42181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2 - ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888 - network_mode: host - extra_hosts: - - "moby:127.0.0.1" - kafka-1: container_name: kafka-1 hostname: kafka-1 - image: confluentinc/cp-kafka:6.2.6 + image: apache/kafka:3.8.0 network_mode: host - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181 + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:19092,CONTROLLER://localhost:19093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 - KAFKA_DEFAULT_REPLICATION_FACTOR: 2 - KAFKA_LOG_RETENTION_HOURS: 120 - KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_COMPRESSION_TYPE: zstd - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" extra_hosts: - "moby:127.0.0.1" kafka-2: container_name: kafka-2 hostname: kafka-2 - image: confluentinc/cp-kafka:6.2.6 + image: apache/kafka:3.8.0 network_mode: host - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 environment: - KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181 + KAFKA_NODE_ID: 2 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 - KAFKA_DEFAULT_REPLICATION_FACTOR: 2 - KAFKA_LOG_RETENTION_HOURS: 120 - KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_COMPRESSION_TYPE: zstd - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" extra_hosts: - "moby:127.0.0.1" kafka-3: container_name: kafka-3 hostname: kafka-3 - image: confluentinc/cp-kafka:6.2.6 + image: apache/kafka:3.8.0 network_mode: host - depends_on: - - zookeeper-1 - - zookeeper-2 - - zookeeper-3 environment: - KAFKA_BROKER_ID: 3 - KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181 + KAFKA_NODE_ID: 3 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:39092,CONTROLLER://localhost:39093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:19093,2@localhost:29093,3@localhost:39093 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_NUM_PARTITIONS: 1 - KAFKA_DEFAULT_REPLICATION_FACTOR: 2 - KAFKA_LOG_RETENTION_HOURS: 120 - KAFKA_LOG_RETENTION_BYTES: 26214400 - KAFKA_COMPRESSION_TYPE: zstd - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + extra_hosts: + - "moby:127.0.0.1" + + kafka-connect: + container_name: kafka-connect + hostname: kafka-connect + image: quay.io/debezium/connect:2.7 + network_mode: host + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + environment: + BOOTSTRAP_SERVERS: localhost:19092,localhost:29092,localhost:39092 + GROUP_ID: 1 + CONFIG_STORAGE_TOPIC: connect_configs + OFFSET_STORAGE_TOPIC: connect_offsets + STATUS_STORAGE_TOPIC: connect_statuses + CONFIG_STORAGE_REPLICATION_FACTOR: 2 + OFFSET_STORAGE_REPLICATION_FACTOR: 2 + STATUS_STORAGE_REPLICATION_FACTOR: 2 + KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter + VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false" + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" + REST_ADVERTISED_HOST_NAME: localhost + REST_PORT: 8083 extra_hosts: - "moby:127.0.0.1" diff --git a/local-setup/README.md b/local-setup/README.md index 54b0f69..c4ae1e3 100644 --- a/local-setup/README.md +++ b/local-setup/README.md @@ -3,6 +3,12 @@ This project helps to get you quickly up and running using [Mendix Business Events](https://docs.mendix.com/appstore/modules/business-events/). The main component is a Docker compose file, so you quickly have the local resources available to support using business events. +#whats new +This setup now uses **Apache Kafka 3.8.0** with **KRaft mode**, which means: +- No Zookeeper required - simpler architecture and easier management +- Built-in CDC support through Kafka Connect and Debezium + + Should you need to install Docker on your local machine, you can get downloads and instructions [here](https://docs.docker.com/engine/install/) If you are using `Microsoft's Visual Studio Code`: diff --git a/local-setup/docker-compose.yml b/local-setup/docker-compose.yml index fe8b405..9d16bb3 100644 --- a/local-setup/docker-compose.yml +++ b/local-setup/docker-compose.yml @@ -1,44 +1,47 @@ --- -version: "3.0" services: - zk: - image: confluentinc/cp-zookeeper:6.2.0 - hostname: zk - container_name: zk - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - mem_limit: 200m - restart: always - kafka: - image: confluentinc/cp-kafka:6.2.0 + image: apache/kafka:3.8.0 hostname: kafka container_name: kafka ports: - "9092:9092" - "9094:9094" - depends_on: - - zk environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zk:2181 + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://kafka:19092,CONTROLLER://kafka:9093,PE://0.0.0.0:9092,LISTENER_P://0.0.0.0:9094 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:19092,PE://127.0.0.1:9092,LISTENER_P://10.211.55.2:9094 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_COMPRESSION_TYPE: lz4 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_DELETE_TOPIC_ENABLE: "true" - KAFKA_MIN_INSYNC_REPLICAS: 1 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PE:PLAINTEXT,LISTENER_P:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PE:PLAINTEXT,LISTENER_P:PLAINTEXT - CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + KAFKA_NUM_PARTITIONS: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" mem_limit: 1500m restart: always + + kafka-connect: + image: quay.io/debezium/connect:2.7 + hostname: kafka-connect + container_name: kafka-connect + ports: + - "8083:8083" + depends_on: + - kafka + environment: + BOOTSTRAP_SERVERS: kafka:19092 + GROUP_ID: 1 + CONFIG_STORAGE_TOPIC: connect_configs + OFFSET_STORAGE_TOPIC: connect_offsets + STATUS_STORAGE_TOPIC: connect_statuses + CONFIG_STORAGE_REPLICATION_FACTOR: 1 + OFFSET_STORAGE_REPLICATION_FACTOR: 1 + STATUS_STORAGE_REPLICATION_FACTOR: 1 + KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter + VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false" + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" + mem_limit: 1000m + restart: always