Skip to content

Commit ec1f39b

Browse files
author
ramk
committed
ATLAS-5336: Upgrade Kafka to 3.9.1; embedded notification broker uses KRaft
Bump kafka-clients and embedded test broker from 2.8.2 to 3.9.1 (Scala 2.13), replace ZooKeeper-based EmbeddedKafkaServer with KafkaClusterTestKit, and update docker dev Kafka image packaging to kafka_2.13-3.9.1.
1 parent 99fc73c commit ec1f39b

9 files changed

Lines changed: 155 additions & 110 deletions

File tree

dev-support/atlas-docker/.env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ ATLAS_VERSION=3.0.0-SNAPSHOT
2121
UBUNTU_VERSION=20.04
2222
HADOOP_VERSION=3.4.2
2323
HBASE_VERSION=2.6.4
24-
KAFKA_VERSION=2.8.2
24+
KAFKA_VERSION=3.9.1
2525
HIVE_VERSION=4.1.0
2626

2727
ATLAS_BACKEND=hbase

dev-support/atlas-docker/Dockerfile.atlas-kafka

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,18 @@ FROM atlas-base:latest
1818

1919
ARG ATLAS_VERSION
2020
ARG KAFKA_VERSION
21+
ARG KAFKA_SCALA_VERSION=2.13
2122

2223
COPY ./dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/
23-
COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
24+
COPY ./downloads/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz /home/atlas/dist/
2425

2526
COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
2627
COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
2728
COPY ./scripts/atlas-kafka-application.properties /home/atlas/scripts/
2829

29-
RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
30-
ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
31-
rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
30+
RUN tar xvfz /home/atlas/dist/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz --directory=/opt/ && \
31+
ln -s /opt/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka && \
32+
rm -f /home/atlas/dist/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz && \
3233
tar xvfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz --directory=/opt/ && \
3334
ln -s /opt/apache-atlas-kafka-hook-${ATLAS_VERSION} /opt/apache-atlas-kafka-hook && \
3435
rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz && \

dev-support/atlas-docker/download-archives.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,6 @@ downloadIfNotPresent() {
4646

4747
downloadIfNotPresent hadoop-${HADOOP_VERSION}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}
4848
downloadIfNotPresent hbase-${HBASE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hbase/${HBASE_VERSION}
49-
downloadIfNotPresent kafka_2.12-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
49+
downloadIfNotPresent kafka_2.13-${KAFKA_VERSION}.tgz https://archive.apache.org/dist/kafka/${KAFKA_VERSION}
5050
downloadIfNotPresent apache-hive-${HIVE_VERSION}-bin.tar.gz https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}
5151

distro/src/conf/atlas-application.properties

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,8 @@ atlas.graph.index.search.max-result-set-size=150
9090
######### Notification Configs #########
9191
atlas.notification.embedded=true
9292
atlas.kafka.data=${sys:atlas.home}/data/kafka
93-
atlas.kafka.zookeeper.connect=localhost:9026
93+
# Embedded broker uses KRaft (no ZooKeeper). bootstrap.servers is updated at runtime when embedded=true.
9494
atlas.kafka.bootstrap.servers=localhost:9027
95-
atlas.kafka.zookeeper.session.timeout.ms=400
96-
atlas.kafka.zookeeper.connection.timeout.ms=200
97-
atlas.kafka.zookeeper.sync.time.ms=20
9895
atlas.kafka.auto.commit.interval.ms=1000
9996
atlas.kafka.hook.group.id=atlas
10097

notification/pom.xml

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,48 @@
6363
<groupId>org.apache.kafka</groupId>
6464
<artifactId>kafka-clients</artifactId>
6565
</dependency>
66-
66+
<dependency>
67+
<groupId>org.apache.kafka</groupId>
68+
<artifactId>kafka-clients</artifactId>
69+
<version>${kafka.version}</version>
70+
<classifier>test</classifier>
71+
</dependency>
72+
<dependency>
73+
<groupId>org.apache.kafka</groupId>
74+
<artifactId>kafka-metadata</artifactId>
75+
<version>${kafka.version}</version>
76+
</dependency>
77+
<dependency>
78+
<groupId>org.apache.kafka</groupId>
79+
<artifactId>kafka-server</artifactId>
80+
<version>${kafka.version}</version>
81+
</dependency>
82+
<dependency>
83+
<groupId>org.apache.kafka</groupId>
84+
<artifactId>kafka-server-common</artifactId>
85+
<version>${kafka.version}</version>
86+
</dependency>
87+
<dependency>
88+
<groupId>org.apache.kafka</groupId>
89+
<artifactId>kafka-server-common</artifactId>
90+
<version>${kafka.version}</version>
91+
<classifier>test</classifier>
92+
</dependency>
93+
<dependency>
94+
<groupId>org.apache.kafka</groupId>
95+
<artifactId>kafka_${kafka.scala.binary.version}</artifactId>
96+
</dependency>
6797
<dependency>
6898
<groupId>org.apache.kafka</groupId>
6999
<artifactId>kafka_${kafka.scala.binary.version}</artifactId>
100+
<version>${kafka.version}</version>
101+
<classifier>test</classifier>
102+
</dependency>
103+
104+
<dependency>
105+
<groupId>org.junit.jupiter</groupId>
106+
<artifactId>junit-jupiter-api</artifactId>
107+
<version>5.11.4</version>
70108
</dependency>
71109

72110
<dependency>
@@ -172,7 +210,7 @@
172210
<artifactItem>
173211
<groupId>org.apache.zookeeper</groupId>
174212
<artifactId>zookeeper</artifactId>
175-
<version>3.4.6</version>
213+
<version>${zookeeper.version}</version>
176214
</artifactItem>
177215
</artifactItems>
178216
</configuration>

notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java

Lines changed: 64 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -17,36 +17,27 @@
1717
*/
1818
package org.apache.atlas.kafka;
1919

20-
import kafka.server.KafkaConfig;
21-
import kafka.server.KafkaServer;
22-
import kafka.zookeeper.ZooKeeperClientException;
20+
import kafka.testkit.KafkaClusterTestKit;
21+
import kafka.testkit.TestKitNodes;
2322
import org.apache.atlas.ApplicationProperties;
2423
import org.apache.atlas.AtlasException;
2524
import org.apache.atlas.service.Service;
26-
import org.apache.atlas.util.CommandHandlerUtility;
2725
import org.apache.commons.configuration2.Configuration;
2826
import org.apache.commons.configuration2.ConfigurationConverter;
27+
import org.apache.kafka.clients.CommonClientConfigs;
2928
import org.apache.kafka.clients.producer.ProducerConfig;
30-
import org.apache.kafka.common.KafkaException;
31-
import org.apache.kafka.common.utils.Time;
32-
import org.apache.zookeeper.server.NIOServerCnxnFactory;
33-
import org.apache.zookeeper.server.ServerCnxnFactory;
34-
import org.apache.zookeeper.server.ZooKeeperServer;
29+
import org.apache.kafka.common.utils.Exit;
30+
import org.apache.kafka.common.utils.Utils;
3531
import org.slf4j.Logger;
3632
import org.slf4j.LoggerFactory;
3733
import org.springframework.core.annotation.Order;
3834
import org.springframework.stereotype.Component;
39-
import scala.Option;
4035

4136
import javax.inject.Inject;
4237

4338
import java.io.File;
44-
import java.io.IOException;
45-
import java.net.BindException;
46-
import java.net.InetSocketAddress;
47-
import java.net.MalformedURLException;
48-
import java.net.URL;
4939
import java.util.Properties;
40+
import java.util.concurrent.atomic.AtomicReference;
5041

5142
@Component
5243
@Order(3)
@@ -56,17 +47,18 @@ public class EmbeddedKafkaServer implements Service {
5647
public static final String PROPERTY_PREFIX = "atlas.kafka";
5748
public static final String PROPERTY_EMBEDDED = "atlas.notification.embedded";
5849

59-
private static final String ATLAS_KAFKA_DATA = "data";
60-
private static final int MAX_RETRY_TO_ACQUIRE_PORT = 3;
50+
private static final String ATLAS_KAFKA_DATA = "data";
6151

62-
private final boolean isEmbedded;
63-
private final Properties properties;
64-
private KafkaServer kafkaServer;
65-
private ServerCnxnFactory factory;
52+
private final boolean isEmbedded;
53+
private final Configuration applicationProperties;
54+
private final Properties properties;
55+
56+
private KafkaClusterTestKit cluster;
6657

6758
@Inject
6859
public EmbeddedKafkaServer(Configuration applicationProperties) {
69-
Configuration kafkaConf = ApplicationProperties.getSubsetConfiguration(applicationProperties, PROPERTY_PREFIX);
60+
this.applicationProperties = applicationProperties;
61+
Configuration kafkaConf = ApplicationProperties.getSubsetConfiguration(applicationProperties, PROPERTY_PREFIX);
7062

7163
this.isEmbedded = applicationProperties.getBoolean(PROPERTY_EMBEDDED, false);
7264
this.properties = ConfigurationConverter.getProperties(kafkaConf);
@@ -78,8 +70,7 @@ public void start() throws AtlasException {
7870

7971
if (isEmbedded) {
8072
try {
81-
startZk();
82-
startKafka();
73+
startKraftBroker();
8374
} catch (Exception e) {
8475
throw new AtlasException("Failed to start embedded kafka", e);
8576
}
@@ -94,90 +85,77 @@ public void start() throws AtlasException {
9485
public void stop() {
9586
LOG.info("==> EmbeddedKafkaServer.stop(isEmbedded={})", isEmbedded);
9687

97-
if (kafkaServer != null) {
98-
kafkaServer.shutdown();
99-
}
88+
if (cluster != null) {
89+
AtomicReference<Throwable> shutdownFailure = new AtomicReference<>();
90+
91+
Utils.closeQuietly(cluster, "embedded Kafka cluster", shutdownFailure);
92+
93+
if (shutdownFailure.get() != null) {
94+
LOG.warn("Failed to shut down embedded Kafka cluster", shutdownFailure.get());
95+
}
10096

101-
if (factory != null) {
102-
factory.shutdown();
97+
cluster = null;
10398
}
10499

105100
LOG.info("<== EmbeddedKafka.stop(isEmbedded={})", isEmbedded);
106101
}
107102

108-
private String startZk() throws IOException, InterruptedException {
109-
String zkValue = properties.getProperty("zookeeper.connect");
103+
private void startKraftBroker() throws Exception {
104+
overrideExitMethods();
110105

111-
LOG.info("Starting zookeeper at {}", zkValue);
106+
File logDir = constructDir("kafka");
112107

113-
URL zkAddress = getURL(zkValue);
114-
File snapshotDir = constructDir("zk/txn");
115-
File logDir = constructDir("zk/snap");
108+
LOG.info("Starting embedded KRaft kafka (log.dir={})", logDir.getAbsolutePath());
116109

117-
for (int attemptCount = 0; attemptCount < MAX_RETRY_TO_ACQUIRE_PORT; attemptCount++) {
118-
try {
119-
factory = NIOServerCnxnFactory.createFactory(new InetSocketAddress(zkAddress.getHost(), zkAddress.getPort()), 1024);
120-
break;
121-
} catch (BindException e) {
122-
LOG.warn("Attempt {}: Starting zookeeper at {} failed", attemptCount, zkValue);
110+
KafkaClusterTestKit.Builder clusterBuilder = new KafkaClusterTestKit.Builder(
111+
new TestKitNodes.Builder()
112+
.setCombined(true)
113+
.setNumBrokerNodes(1)
114+
.setNumControllerNodes(1)
115+
.build());
123116

124-
if (attemptCount == MAX_RETRY_TO_ACQUIRE_PORT - 1) {
125-
throw e;
126-
}
117+
Properties brokerConfig = buildBrokerConfig(logDir);
127118

128-
CommandHandlerUtility.tryKillingProcessUsingPort(zkAddress.getPort(), attemptCount != 0);
129-
}
130-
}
119+
brokerConfig.forEach((key, value) -> clusterBuilder.setConfigProp(key.toString(), value));
131120

132-
factory.startup(new ZooKeeperServer(snapshotDir, logDir, 500));
121+
cluster = clusterBuilder.build();
122+
cluster.format();
123+
cluster.startup();
124+
cluster.waitForReadyBrokers();
133125

134-
String ret = factory.getLocalAddress().getAddress().toString();
126+
String bootstrapServers = cluster.clientProperties().get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG).toString();
135127

136-
LOG.info("Embedded zookeeper for Kafka started at {}", ret);
128+
LOG.info("Embedded KRaft kafka server started at {}", bootstrapServers);
137129

138-
return ret;
130+
applicationProperties.setProperty(PROPERTY_PREFIX + ".bootstrap.servers", bootstrapServers);
131+
properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
139132
}
140133

141-
private void startKafka() throws IOException {
142-
String kafkaValue = properties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
134+
private Properties buildBrokerConfig(File logDir) {
135+
Properties brokerConfig = new Properties();
143136

144-
LOG.info("Starting kafka at {}", kafkaValue);
137+
brokerConfig.setProperty("log.dir", logDir.getAbsolutePath());
138+
brokerConfig.setProperty("delete.topic.enable", "true");
139+
brokerConfig.setProperty("group.initial.rebalance.delay.ms", "0");
140+
brokerConfig.setProperty("offsets.topic.replication.factor", "1");
141+
brokerConfig.setProperty("transaction.state.log.replication.factor", "1");
142+
brokerConfig.setProperty("transaction.state.log.min.isr", "1");
143+
brokerConfig.setProperty("num.partitions", "1");
145144

146-
URL kafkaAddress = getURL(kafkaValue);
147-
Properties brokerConfig = properties;
145+
String replicationFactor = properties.getProperty("offsets.topic.replication.factor");
148146

149-
for (int attemptCount = 0; attemptCount < MAX_RETRY_TO_ACQUIRE_PORT; attemptCount++) {
150-
try {
151-
brokerConfig.setProperty("broker.id", "1");
152-
brokerConfig.setProperty("host.name", kafkaAddress.getHost());
153-
brokerConfig.setProperty("port", String.valueOf(kafkaAddress.getPort()));
154-
brokerConfig.setProperty("log.dirs", constructDir("kafka").getAbsolutePath());
155-
brokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));
156-
157-
kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), Time.SYSTEM, Option.apply(this.getClass().getName()), false);
158-
159-
kafkaServer.startup();
160-
break;
161-
} catch (KafkaException | ZooKeeperClientException e) {
162-
LOG.warn("Attempt {}: kafka server with broker config {} failed", attemptCount, brokerConfig);
163-
164-
if (attemptCount == MAX_RETRY_TO_ACQUIRE_PORT - 1) {
165-
throw e;
166-
}
167-
168-
if (kafkaServer != null) {
169-
try {
170-
kafkaServer.shutdown();
171-
} catch (Exception ex) {
172-
LOG.info("Failed to shutdown kafka server", ex);
173-
}
174-
}
175-
176-
CommandHandlerUtility.tryKillingProcessUsingPort(kafkaAddress.getPort(), attemptCount != 0);
177-
}
147+
if (replicationFactor != null) {
148+
brokerConfig.setProperty("offsets.topic.replication.factor", replicationFactor);
178149
}
179150

180-
LOG.info("Embedded kafka server started with broker config {}", brokerConfig);
151+
return brokerConfig;
152+
}
153+
154+
private void overrideExitMethods() {
155+
Exit.setExitProcedure((statusCode, message) ->
156+
LOG.warn("Kafka Exit.exit({}, {}) suppressed in embedded broker", statusCode, message));
157+
Exit.setHaltProcedure((statusCode, message) ->
158+
LOG.warn("Kafka Exit.halt({}, {}) suppressed in embedded broker", statusCode, message));
181159
}
182160

183161
private File constructDir(String dirPrefix) {
@@ -189,12 +167,4 @@ private File constructDir(String dirPrefix) {
189167

190168
return file;
191169
}
192-
193-
private URL getURL(String url) throws MalformedURLException {
194-
try {
195-
return new URL(url);
196-
} catch (MalformedURLException e) {
197-
return new URL("http://" + url);
198-
}
199-
}
200170
}

notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,20 @@ public static List<String> trimAndPurge(String[] strings) {
174174
public void start() throws AtlasException {
175175
LOG.info("==> KafkaNotification.start()");
176176

177+
syncBootstrapServers();
178+
177179
LOG.info("<== KafkaNotification.start()");
178180
}
179181

182+
private void syncBootstrapServers() throws AtlasException {
183+
Configuration kafkaConf = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(), PROPERTY_PREFIX);
184+
String bootstrap = kafkaConf.getString("bootstrap.servers");
185+
186+
if (StringUtils.isNotEmpty(bootstrap)) {
187+
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap);
188+
}
189+
}
190+
180191
// ----- Service ---------------------------------------------------------
181192

182193
@Override

pom.xml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,8 @@
130130
<httpcomponents-httpclient.version>4.5.13</httpcomponents-httpclient.version>
131131
<httpcomponents-httpcore.version>4.4.13</httpcomponents-httpcore.version>
132132
<ivy.version>2.5.2</ivy.version>
133-
<jackson.databind.version>2.12.7</jackson.databind.version>
134-
<jackson.version>2.12.7</jackson.version>
133+
<jackson.databind.version>2.16.2</jackson.databind.version>
134+
<jackson.version>2.16.2</jackson.version>
135135
<jacoco.version>0.8.13</jacoco.version>
136136
<janusgraph.cassandra.version>0.5.3</janusgraph.cassandra.version>
137137
<janusgraph.version>1.1.0</janusgraph.version>
@@ -149,8 +149,8 @@
149149
<json.version>3.2.11</json.version>
150150
<jsr.version>1.1</jsr.version>
151151
<junit.version>4.13.2</junit.version>
152-
<kafka.scala.binary.version>2.12</kafka.scala.binary.version>
153-
<kafka.version>2.8.2</kafka.version>
152+
<kafka.scala.binary.version>2.13</kafka.scala.binary.version>
153+
<kafka.version>3.9.1</kafka.version>
154154
<keycloak.version>6.0.1</keycloak.version>
155155
<logback.version>1.3.15</logback.version>
156156
<lucene-solr.version>8.11.3</lucene-solr.version>

0 commit comments

Comments
 (0)