Skip to content

Commit 4819744

Browse files
maltesandermaltesander
committed
add kafka 4.0.0 and log4j2
1 parent c1d1dfa commit 4819744

6 files changed

Lines changed: 106 additions & 19 deletions

File tree

‎rust/operator-binary/src/product_logging.rs‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,16 @@ const CONSOLE_CONVERSION_PATTERN_LOG4J: &str = "[%d] %p %m (%c)%n";
3333
const CONSOLE_CONVERSION_PATTERN_LOG4J2: &str = "%d{ISO8601} %p [%t] %c - %m%n";
3434

3535
pub fn kafka_log_opts(product_version: &str) -> String {
36-
if product_version.starts_with("4.") {
37-
format!("-Dlog4j2.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}")
38-
} else {
36+
if product_version.starts_with("3.") {
3937
format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}")
38+
} else {
39+
// TODO: -Dlog4j2 vs -Dlog4j
40+
format!("-Dlog4j2.configurationFile=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}")
4041
}
4142
}
4243

43-
pub fn kafka_log_opts_env_var(product_version: &str) -> String {
44-
if product_version.starts_with("4.") {
45-
"KAFKA_LOG4J2_OPTS".to_string()
46-
} else {
47-
"KAFKA_LOG4J_OPTS".to_string()
48-
}
44+
pub fn kafka_log_opts_env_var() -> String {
45+
"KAFKA_LOG4J_OPTS".to_string()
4946
}
5047

5148
/// Extend the role group ConfigMap with logging and Vector configurations
@@ -61,21 +58,21 @@ pub fn extend_role_group_config_map(
6158
};
6259

6360
// Starting with Kafka 4.0, log4j2 is used instead of log4j.
64-
match product_version.starts_with("4.") {
65-
true => add_log4j2_config_if_automatic(
61+
match product_version.starts_with("3.") {
62+
true => add_log4j_config_if_automatic(
6663
cm_builder,
6764
Some(merged_config.kafka_logging()),
68-
LOG4J2_CONFIG_FILE,
65+
LOG4J_CONFIG_FILE,
6966
container_name,
70-
KAFKA_LOG4J2_FILE,
67+
KAFKA_LOG4J_FILE,
7168
MAX_KAFKA_LOG_FILES_SIZE,
7269
),
73-
false => add_log4j_config_if_automatic(
70+
false => add_log4j2_config_if_automatic(
7471
cm_builder,
7572
Some(merged_config.kafka_logging()),
76-
LOG4J_CONFIG_FILE,
73+
LOG4J2_CONFIG_FILE,
7774
container_name,
78-
KAFKA_LOG4J_FILE,
75+
KAFKA_LOG4J2_FILE,
7976
MAX_KAFKA_LOG_FILES_SIZE,
8077
),
8178
}

‎rust/operator-binary/src/resource/statefulset.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ pub fn build_broker_rolegroup_statefulset(
329329
.context(ConstructJvmArgumentsSnafu)?,
330330
)
331331
.add_env_var(
332-
kafka_log_opts_env_var(&resolved_product_image.product_version),
332+
kafka_log_opts_env_var(),
333333
kafka_log_opts(&resolved_product_image.product_version),
334334
)
335335
// Needed for the `containerdebug` process to log it's tracing information to.
@@ -671,7 +671,7 @@ pub fn build_controller_rolegroup_statefulset(
671671
.context(ConstructJvmArgumentsSnafu)?,
672672
)
673673
.add_env_var(
674-
kafka_log_opts_env_var(&resolved_product_image.product_version),
674+
kafka_log_opts_env_var(),
675675
kafka_log_opts(&resolved_product_image.product_version),
676676
)
677677
// Needed for the `containerdebug` process to log it's tracing information to.

tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml renamed to tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml.j2

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,9 @@ kind: TestStep
44
commands:
55
- script: >
66
kubectl create configmap prepared-logs
7+
{% if test_scenario['values']['kafka'].startswith('3.') %}
78
--from-file=prepared-logs.log4j.xml
9+
{% else %}
10+
--from-file=prepared-logs.log4j2.xml
11+
{% endif %}
812
--namespace=$NAMESPACE

‎tests/templates/kuttl/logging/04-install-kafka.yaml.j2‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ kind: ConfigMap
44
metadata:
55
name: kafka-log-config
66
data:
7+
{% if test_scenario['values']['kafka'].startswith('3.') %}
78
log4j.properties: |
89
log4j.rootLogger=INFO, CONSOLE, FILE
910

@@ -18,6 +19,19 @@ data:
1819
log4j.appender.FILE.MaxFileSize=5MB
1920
log4j.appender.FILE.MaxBackupIndex=1
2021
log4j.appender.FILE.layout=org.apache.log4j.xml.XMLLayout
22+
{% else %}
23+
log4j2.properties: |-
24+
appenders = FILE
25+
26+
appender.FILE.type = File
27+
appender.FILE.name = FILE
28+
appender.FILE.fileName = /stackable/log/kafka/kafka.log4j2.xml
29+
appender.FILE.layout.type = XMLLayout
30+
31+
rootLogger.level=INFO
32+
rootLogger.appenderRefs = FILE
33+
rootLogger.appenderRef.FILE.ref = FILE
34+
{% endif %}
2135
---
2236
apiVersion: kafka.stackable.tech/v1alpha1
2337
kind: KafkaCluster

‎tests/templates/kuttl/logging/prepared-logs.log4j2.xml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
2+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
3+
<Message>Valid log event with all possible tags and attributes</Message>
4+
<Thrown name="TestException" localizedMessage="localized test message" message="test message">
5+
<ExtendedStackTrace>
6+
<ExtendedStackTraceItem class="TestClass" method="testMethod1" file="TestFile.java" line="123" exact="false" location="Test.jar" version="1.0.0" />
7+
<ExtendedStackTraceItem class="TestClass" method="testMethod2" file="TestFile.java" line="456" exact="true" location="Test.jar" version="1.0.0" />
8+
</ExtendedStackTrace>
9+
</Thrown>
10+
</Event>
11+
12+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
13+
<Message>Valid log event without the Instant tag</Message>
14+
</Event>
15+
16+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
17+
<Instant nanoOfSecond="000000000"/>
18+
<Message>Invalid log event without epochSecond</Message>
19+
</Event>
20+
21+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
22+
<Instant epochSecond="1704063600"/>
23+
<Message>Invalid log event without nanoOfSecond</Message>
24+
</Event>
25+
26+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
27+
<Instant epochSecond="10000000000000" nanoOfSecond="000000000"/>
28+
<Message>Invalid log event with invalid epochSecond</Message>
29+
</Event>
30+
31+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
32+
<Message>Invalid log event without a timestamp</Message>
33+
</Event>
34+
35+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="10000000000000000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
36+
<Message>Invalid log event with invalid timeMillis</Message>
37+
</Event>
38+
39+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" endOfBatch="false" threadId="1" threadPriority="5">
40+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
41+
<Message>Invalid log event without a logger</Message>
42+
</Event>
43+
44+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
45+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
46+
<Message>Invalid log event without a level</Message>
47+
</Event>
48+
49+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="CRITICAL" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
50+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
51+
<Message>Invalid log event with an unknown level</Message>
52+
</Event>
53+
54+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
55+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
56+
<!-- <Message>Invalid log event without a message</Message> -->
57+
</Event>
58+
59+
<NoEvent xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
60+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
61+
<Message>Invalid log event without the Event tag</Message>
62+
</NoEvent>
63+
64+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
65+
<Message>Unparsable log event</Message>
66+
<Invalid tag ...
67+
</Event>
68+
69+
<Event xmlns="http://logging.apache.org/log4j/2.0/events" timeMillis="1704063600000" thread="main" level="INFO" loggerName="TestLogger" endOfBatch="false" loggerFqcn="TestLogger" threadId="1" threadPriority="5">
70+
<Instant epochSecond="1704063600" nanoOfSecond="000000000"/>
71+
<Message>Valid log event after the unparsable one</Message>
72+
</Event>

‎tests/test-definition.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ dimensions:
77
- name: kafka
88
values:
99
- 3.7.2
10-
- 3.9.0
1110
- 3.9.1
11+
- 4.0.0
1212
# Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
1313
# as in the example below.
1414
# - 3.8.0,oci.stackable.tech/sdp/kafka:3.8.0-stackable0.0.0-dev

0 commit comments

Comments
 (0)