Skip to content

Commit 9726dd1

Browse files
committed
Enhance spring-kafka plugin to support kafka-clients 3.7.1+ with spring-kafka 3.1.0+
1 parent 53a00a6 commit 9726dd1

13 files changed

Lines changed: 576 additions & 2 deletions

File tree

CHANGES.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ Release Notes.
2020
* Eliminate repeated code with HttpServletRequestWrapper in mvc-annotation-commons.
2121
* Add the jdk httpclient plugin.
2222
* Fix Gateway 2.0.x plugin not activated for spring-cloud-starter-gateway 2.0.0.RELEASE.
23+
* Enhance spring-kafka plugin to support kafka-clients 3.7.1+ with spring-kafka 3.1.0+
2324
All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/242?closed=1)
2425

2526
------------------

apm-sniffer/apm-sdk-plugin/spring-plugins/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
<module>spring-commons</module>
4141
<module>mvc-annotation-5.x-plugin</module>
4242
<module>spring-kafka-1.x-plugin</module>
43-
<module>spring-kafka-2.x-plugin</module>
43+
<module>spring-kafka-2.x-3.x-plugin</module>
4444
<module>scheduled-annotation-plugin</module>
4545
<module>spring-webflux-5.x-webclient-plugin</module>
4646
<module>spring-webflux-6.x-webclient-plugin</module>

apm-sniffer/apm-sdk-plugin/spring-plugins/spring-kafka-2.x-plugin/pom.xml renamed to apm-sniffer/apm-sdk-plugin/spring-plugins/spring-kafka-2.x-3.x-plugin/pom.xml

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@
2525
<version>9.6.0-SNAPSHOT</version>
2626
</parent>
2727

28-
<artifactId>apm-spring-kafka-2.x-plugin</artifactId>
28+
<artifactId>apm-spring-kafka-2.x-3.x-plugin</artifactId>
2929

3030
<properties>
3131
<spring-kafka.version>2.2.9.RELEASE</spring-kafka.version>
32+
<kafka-clients.version>3.9.1</kafka-clients.version>
3233
</properties>
3334

3435
<dependencies>
@@ -44,5 +45,29 @@
4445
<version>${project.version}</version>
4546
<scope>provided</scope>
4647
</dependency>
48+
<dependency>
49+
<groupId>org.apache.kafka</groupId>
50+
<artifactId>kafka-clients</artifactId>
51+
<version>${kafka-clients.version}</version>
52+
<scope>provided</scope>
53+
</dependency>
54+
55+
<!-- Test dependencies -->
56+
<dependency>
57+
<groupId>org.apache.skywalking</groupId>
58+
<artifactId>apm-test-tools</artifactId>
59+
<version>${project.version}</version>
60+
<scope>test</scope>
61+
</dependency>
62+
<dependency>
63+
<groupId>org.mockito</groupId>
64+
<artifactId>mockito-core</artifactId>
65+
<scope>test</scope>
66+
</dependency>
67+
<dependency>
68+
<groupId>junit</groupId>
69+
<artifactId>junit</artifactId>
70+
<scope>test</scope>
71+
</dependency>
4772
</dependencies>
4873
</project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.apm.plugin.spring.kafka;
20+
21+
import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance;
22+
import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceConstructorInterceptor;
23+
import org.apache.skywalking.apm.util.StringUtil;
24+
25+
public class ExtendedConstructorInterceptPoint implements InstanceConstructorInterceptor {
26+
@Override
27+
public void onConstruct(final EnhancedInstance objInst, final Object[] allArguments) throws Throwable {
28+
ExtendedConsumerEnhanceRequiredInfo requiredInfo = new ExtendedConsumerEnhanceRequiredInfo();
29+
extractConsumerConfig(allArguments, requiredInfo);
30+
objInst.setSkyWalkingDynamicField(requiredInfo);
31+
}
32+
33+
private void extractConsumerConfig(Object[] allArguments, ExtendedConsumerEnhanceRequiredInfo requiredInfo) {
34+
if (allArguments == null || allArguments.length == 0) {
35+
return;
36+
}
37+
38+
for (Object arg : allArguments) {
39+
if (arg instanceof java.util.Map) {
40+
extractConfigFromMap(arg, requiredInfo);
41+
break;
42+
}
43+
}
44+
}
45+
46+
private void extractConfigFromMap(Object arg, ExtendedConsumerEnhanceRequiredInfo requiredInfo) {
47+
try {
48+
java.util.Map<String, Object> configMap = (java.util.Map<String, Object>) arg;
49+
Object bootstrapServers = configMap.get("bootstrap.servers");
50+
if (bootstrapServers instanceof java.util.List) {
51+
requiredInfo.setBrokerServers(StringUtil.join(';', String.valueOf(bootstrapServers)));
52+
} else if (bootstrapServers != null) {
53+
requiredInfo.setBrokerServers(bootstrapServers.toString());
54+
}
55+
56+
Object groupId = configMap.get("group.id");
57+
if (groupId != null) {
58+
requiredInfo.setGroupId(groupId.toString());
59+
}
60+
} catch (Exception e) {
61+
// Ignore exception and continue
62+
}
63+
}
64+
65+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.apm.plugin.spring.kafka;
20+
21+
/**
22+
* Extended ConsumerEnhanceRequiredInfo to hold additional information for ExtendedKafkaConsumer
23+
*/
24+
25+
public class ExtendedConsumerEnhanceRequiredInfo {
26+
27+
private static final String UNKNOWN = "Unknown";
28+
29+
private String brokerServers = UNKNOWN;
30+
private String groupId = UNKNOWN;
31+
private long startTime;
32+
33+
public void setBrokerServers(String brokerServers) {
34+
this.brokerServers = brokerServers != null ? brokerServers : UNKNOWN;
35+
}
36+
37+
public String getBrokerServers() {
38+
return brokerServers;
39+
}
40+
41+
public void setGroupId(String groupId) {
42+
this.groupId = groupId != null ? groupId : UNKNOWN;
43+
}
44+
45+
public String getGroupId() {
46+
return groupId;
47+
}
48+
49+
public void setStartTime(long startTime) {
50+
this.startTime = startTime;
51+
}
52+
53+
public long getStartTime() {
54+
return startTime;
55+
}
56+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.apm.plugin.spring.kafka;
20+
21+
import java.lang.reflect.Method;
22+
import java.nio.charset.StandardCharsets;
23+
import java.util.Iterator;
24+
import java.util.Set;
25+
import java.util.stream.Collectors;
26+
import org.apache.kafka.clients.consumer.ConsumerRecord;
27+
import org.apache.kafka.clients.consumer.ConsumerRecords;
28+
import org.apache.kafka.common.TopicPartition;
29+
import org.apache.kafka.common.header.Header;
30+
import org.apache.skywalking.apm.agent.core.context.CarrierItem;
31+
import org.apache.skywalking.apm.agent.core.context.ContextCarrier;
32+
import org.apache.skywalking.apm.agent.core.context.ContextManager;
33+
import org.apache.skywalking.apm.agent.core.context.tag.Tags;
34+
import org.apache.skywalking.apm.agent.core.context.trace.AbstractSpan;
35+
import org.apache.skywalking.apm.agent.core.context.trace.SpanLayer;
36+
import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance;
37+
import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceMethodsAroundInterceptor;
38+
import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.MethodInterceptResult;
39+
import org.apache.skywalking.apm.network.trace.component.ComponentsDefine;
40+
import org.apache.skywalking.apm.plugin.kafka.define.Constants;
41+
import org.apache.skywalking.apm.plugin.kafka.define.KafkaContext;
42+
43+
public class ExtendedKafkaConsumerInterceptor implements InstanceMethodsAroundInterceptor {
44+
45+
private static final String OPERATE_NAME_PREFIX = "Kafka/";
46+
private static final String CONSUMER_OPERATE_NAME = "/Consumer/";
47+
private static final String UNKNOWN = "Unknown";
48+
49+
@Override
50+
public void beforeMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class<?>[] argumentsTypes,
51+
MethodInterceptResult result) throws Throwable {
52+
ExtendedConsumerEnhanceRequiredInfo requiredInfo = (ExtendedConsumerEnhanceRequiredInfo) objInst.getSkyWalkingDynamicField();
53+
if (requiredInfo != null) {
54+
requiredInfo.setStartTime(System.currentTimeMillis());
55+
}
56+
}
57+
58+
@Override
59+
public Object afterMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class<?>[] argumentsTypes,
60+
Object ret) throws Throwable {
61+
if (ret == null) {
62+
return ret;
63+
}
64+
65+
ConsumerRecords<?, ?> records = (ConsumerRecords<?, ?>) ret;
66+
67+
// Only create entry span when consumer received at least one message
68+
if (records.count() > 0) {
69+
createEntrySpan(objInst, records);
70+
}
71+
return ret;
72+
}
73+
74+
@Override
75+
public void handleMethodException(EnhancedInstance objInst, Method method, Object[] allArguments,
76+
Class<?>[] argumentsTypes, Throwable t) {
77+
if (ContextManager.isActive()) {
78+
ContextManager.activeSpan().log(t);
79+
}
80+
}
81+
82+
private void createEntrySpan(EnhancedInstance objInst, ConsumerRecords<?, ?> records) {
83+
KafkaContext context = (KafkaContext) ContextManager.getRuntimeContext().get(Constants.KAFKA_FLAG);
84+
if (context != null) {
85+
ContextManager.createEntrySpan(context.getOperationName(), null);
86+
context.setNeedStop(true);
87+
}
88+
89+
ExtendedConsumerEnhanceRequiredInfo requiredInfo = (ExtendedConsumerEnhanceRequiredInfo) objInst.getSkyWalkingDynamicField();
90+
91+
SpanInfo spanInfo = buildSpanInfo(requiredInfo, records);
92+
93+
String operationName = OPERATE_NAME_PREFIX + spanInfo.topic + CONSUMER_OPERATE_NAME + spanInfo.groupId;
94+
AbstractSpan activeSpan = ContextManager.createEntrySpan(operationName, null);
95+
96+
if (requiredInfo != null) {
97+
activeSpan.start(requiredInfo.getStartTime());
98+
}
99+
100+
activeSpan.setComponent(ComponentsDefine.KAFKA_CONSUMER);
101+
SpanLayer.asMQ(activeSpan);
102+
Tags.MQ_BROKER.set(activeSpan, spanInfo.brokerServers);
103+
Tags.MQ_TOPIC.set(activeSpan, spanInfo.topic);
104+
activeSpan.setPeer(spanInfo.brokerServers);
105+
106+
extractContextCarrier(records);
107+
ContextManager.stopSpan();
108+
}
109+
110+
private SpanInfo buildSpanInfo(ExtendedConsumerEnhanceRequiredInfo requiredInfo, ConsumerRecords<?, ?> records) {
111+
String topic = UNKNOWN;
112+
String groupId = UNKNOWN;
113+
String brokerServers = UNKNOWN;
114+
115+
if (requiredInfo != null) {
116+
groupId = requiredInfo.getGroupId();
117+
brokerServers = requiredInfo.getBrokerServers();
118+
}
119+
120+
Set<TopicPartition> partitions = records.partitions();
121+
if (!partitions.isEmpty()) {
122+
topic = partitions.stream()
123+
.map(TopicPartition::topic).distinct()
124+
.collect(Collectors.joining(";"));
125+
}
126+
127+
return new SpanInfo(topic, groupId, brokerServers);
128+
}
129+
130+
private void extractContextCarrier(ConsumerRecords<?, ?> records) {
131+
for (ConsumerRecord<?, ?> record : records) {
132+
ContextCarrier contextCarrier = new ContextCarrier();
133+
CarrierItem next = contextCarrier.items();
134+
135+
while (next.hasNext()) {
136+
next = next.next();
137+
Iterator<Header> iterator = record.headers().headers(next.getHeadKey()).iterator();
138+
if (iterator.hasNext()) {
139+
next.setHeadValue(new String(iterator.next().value(), StandardCharsets.UTF_8));
140+
}
141+
}
142+
ContextManager.extract(contextCarrier);
143+
}
144+
}
145+
146+
private static class SpanInfo {
147+
final String topic;
148+
final String groupId;
149+
final String brokerServers;
150+
151+
SpanInfo(String topic, String groupId, String brokerServers) {
152+
this.topic = topic;
153+
this.groupId = groupId;
154+
this.brokerServers = brokerServers;
155+
}
156+
}
157+
}

apm-sniffer/apm-sdk-plugin/spring-plugins/spring-kafka-2.x-plugin/src/main/java/org/apache/skywalking/apm/plugin/spring/kafka/KafkaTemplateCallbackInterceptor.java renamed to apm-sniffer/apm-sdk-plugin/spring-plugins/spring-kafka-2.x-3.x-plugin/src/main/java/org/apache/skywalking/apm/plugin/spring/kafka/KafkaTemplateCallbackInterceptor.java

File renamed without changes.

apm-sniffer/apm-sdk-plugin/spring-plugins/spring-kafka-2.x-plugin/src/main/java/org/apache/skywalking/apm/plugin/spring/kafka/PollAndInvokeMethodInterceptor.java renamed to apm-sniffer/apm-sdk-plugin/spring-plugins/spring-kafka-2.x-3.x-plugin/src/main/java/org/apache/skywalking/apm/plugin/spring/kafka/PollAndInvokeMethodInterceptor.java

File renamed without changes.

0 commit comments

Comments
 (0)