Skip to content

Commit 87809b5

Browse files
maltesandermaltesander
committed
prepare log4j2 config
1 parent d5ee2fd commit 87809b5

5 files changed

Lines changed: 136 additions & 62 deletions

File tree

‎rust/operator-binary/src/config/command.rs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,21 @@ use stackable_operator::{
66
utils::COMMON_BASH_TRAP_FUNCTIONS,
77
};
88

9-
use crate::crd::{
10-
KafkaPodDescriptor, STACKABLE_CONFIG_DIR, STACKABLE_KERBEROS_KRB5_PATH,
11-
STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, STACKABLE_LOG_DIR,
12-
listener::{KafkaListenerConfig, KafkaListenerName, node_address_cmd},
13-
role::{
14-
KAFKA_ADVERTISED_LISTENERS, KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS,
15-
KAFKA_CONTROLLER_QUORUM_VOTERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS,
16-
KAFKA_NODE_ID, KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BROKER_PROPERTIES_FILE,
17-
controller::CONTROLLER_PROPERTIES_FILE,
9+
use crate::{
10+
crd::{
11+
KafkaPodDescriptor, STACKABLE_CONFIG_DIR, STACKABLE_KERBEROS_KRB5_PATH,
12+
STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR,
13+
listener::{KafkaListenerConfig, KafkaListenerName, node_address_cmd},
14+
role::{
15+
KAFKA_ADVERTISED_LISTENERS, KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS,
16+
KAFKA_CONTROLLER_QUORUM_VOTERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS,
17+
KAFKA_NODE_ID, KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BROKER_PROPERTIES_FILE,
18+
controller::CONTROLLER_PROPERTIES_FILE,
19+
},
20+
security::KafkaTlsSecurity,
21+
v1alpha1,
1822
},
19-
security::KafkaTlsSecurity,
20-
v1alpha1,
23+
product_logging::STACKABLE_LOG_DIR,
2124
};
2225

2326
/// Returns the commands to start the main Kafka container
@@ -45,7 +48,7 @@ pub fn broker_kafka_container_commands(
4548
remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR),
4649
create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR),
4750
set_realm_env = match kafka_security.has_kerberos_enabled() {
48-
true => format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {})", STACKABLE_KERBEROS_KRB5_PATH),
51+
true => format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH})"),
4952
false => "".to_string(),
5053
},
5154
broker_start_command = broker_start_command(kafka, cluster_id, controller_descriptors, kafka_listeners, opa_connect_string, kafka_security, product_version),

‎rust/operator-binary/src/crd/mod.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,6 @@ pub const STACKABLE_LISTENER_BROKER_DIR: &str = "/stackable/listener-broker";
5252
pub const STACKABLE_LISTENER_BOOTSTRAP_DIR: &str = "/stackable/listener-bootstrap";
5353
pub const STACKABLE_DATA_DIR: &str = "/stackable/data";
5454
pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config";
55-
pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config";
56-
pub const STACKABLE_LOG_DIR: &str = "/stackable/log";
5755
// kerberos
5856
pub const STACKABLE_KERBEROS_DIR: &str = "/stackable/kerberos";
5957
pub const STACKABLE_KERBEROS_KRB5_PATH: &str = "/stackable/kerberos/krb5.conf";

‎rust/operator-binary/src/product_logging.rs‎

Lines changed: 106 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -11,65 +11,74 @@ use stackable_operator::{
1111
};
1212

1313
use crate::crd::{
14-
STACKABLE_LOG_DIR,
1514
role::{AnyConfig, broker::BrokerContainer, controller::ControllerContainer},
1615
v1alpha1,
1716
};
1817

18+
pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config";
19+
pub const STACKABLE_LOG_DIR: &str = "/stackable/log";
20+
// log4j
1921
pub const LOG4J_CONFIG_FILE: &str = "log4j.properties";
20-
pub const KAFKA_LOG_FILE: &str = "kafka.log4j.xml";
21-
22+
pub const KAFKA_LOG4J_FILE: &str = "kafka.log4j.xml";
23+
// log4j2
24+
pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties";
25+
pub const KAFKA_LOG4J2_FILE: &str = "kafka.log4j2.xml";
26+
// max size
2227
pub const MAX_KAFKA_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity {
2328
value: 10.0,
2429
unit: BinaryMultiple::Mebi,
2530
};
2631

27-
const CONSOLE_CONVERSION_PATTERN: &str = "[%d] %p %m (%c)%n";
32+
const CONSOLE_CONVERSION_PATTERN_LOG4J: &str = "[%d] %p %m (%c)%n";
33+
const CONSOLE_CONVERSION_PATTERN_LOG4J2: &str = "%d{ISO8601} %p [%t] %c - %m%n";
34+
35+
pub fn kafka_log_opts(product_version: &str) -> String {
36+
if product_version.starts_with("4.") {
37+
format!("-Dlog4j2.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J2_CONFIG_FILE}")
38+
} else {
39+
format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}")
40+
}
41+
}
42+
43+
pub fn kafka_log_opts_env_var(product_version: &str) -> String {
44+
if product_version.starts_with("4.") {
45+
"KAFKA_LOG4J2_OPTS".to_string()
46+
} else {
47+
"KAFKA_LOG4J_OPTS".to_string()
48+
}
49+
}
2850

2951
/// Extend the role group ConfigMap with logging and Vector configurations
3052
pub fn extend_role_group_config_map(
53+
product_version: &str,
3154
rolegroup: &RoleGroupRef<v1alpha1::KafkaCluster>,
3255
merged_config: &AnyConfig,
3356
cm_builder: &mut ConfigMapBuilder,
3457
) {
35-
fn add_log4j_config_if_automatic(
36-
cm_builder: &mut ConfigMapBuilder,
37-
log_config: Option<Cow<ContainerLogConfig>>,
38-
log_config_file: &str,
39-
container_name: impl Display,
40-
log_file: &str,
41-
max_log_file_size: MemoryQuantity,
42-
) {
43-
if let Some(ContainerLogConfig {
44-
choice: Some(ContainerLogConfigChoice::Automatic(log_config)),
45-
}) = log_config.as_deref()
46-
{
47-
cm_builder.add_data(
48-
log_config_file,
49-
product_logging::framework::create_log4j_config(
50-
&format!("{STACKABLE_LOG_DIR}/{container_name}"),
51-
log_file,
52-
max_log_file_size
53-
.scale_to(BinaryMultiple::Mebi)
54-
.floor()
55-
.value as u32,
56-
CONSOLE_CONVERSION_PATTERN,
57-
log_config,
58-
),
59-
);
60-
}
58+
let container_name = match merged_config {
59+
AnyConfig::Broker(_) => BrokerContainer::Kafka.to_string(),
60+
AnyConfig::Controller(_) => ControllerContainer::Kafka.to_string(),
61+
};
62+
63+
// Starting with Kafka 4.0, log4j2 is used instead of log4j.
64+
match product_version.starts_with("4.") {
65+
true => add_log4j2_config_if_automatic(
66+
cm_builder,
67+
Some(merged_config.kafka_logging()),
68+
LOG4J2_CONFIG_FILE,
69+
container_name,
70+
KAFKA_LOG4J2_FILE,
71+
MAX_KAFKA_LOG_FILES_SIZE,
72+
),
73+
false => add_log4j_config_if_automatic(
74+
cm_builder,
75+
Some(merged_config.kafka_logging()),
76+
LOG4J_CONFIG_FILE,
77+
container_name,
78+
KAFKA_LOG4J_FILE,
79+
MAX_KAFKA_LOG_FILES_SIZE,
80+
),
6181
}
62-
add_log4j_config_if_automatic(
63-
cm_builder,
64-
Some(merged_config.kafka_logging()),
65-
LOG4J_CONFIG_FILE,
66-
match merged_config {
67-
AnyConfig::Broker(_) => BrokerContainer::Kafka.to_string(),
68-
AnyConfig::Controller(_) => ControllerContainer::Kafka.to_string(),
69-
},
70-
KAFKA_LOG_FILE,
71-
MAX_KAFKA_LOG_FILES_SIZE,
72-
);
7382

7483
let vector_log_config = merged_config.vector_logging();
7584
let vector_log_config = if let ContainerLogConfig {
@@ -88,3 +97,59 @@ pub fn extend_role_group_config_map(
8897
);
8998
}
9099
}
100+
101+
fn add_log4j_config_if_automatic(
102+
cm_builder: &mut ConfigMapBuilder,
103+
log_config: Option<Cow<ContainerLogConfig>>,
104+
log_config_file: &str,
105+
container_name: impl Display,
106+
log_file: &str,
107+
max_log_file_size: MemoryQuantity,
108+
) {
109+
if let Some(ContainerLogConfig {
110+
choice: Some(ContainerLogConfigChoice::Automatic(log_config)),
111+
}) = log_config.as_deref()
112+
{
113+
cm_builder.add_data(
114+
log_config_file,
115+
product_logging::framework::create_log4j_config(
116+
&format!("{STACKABLE_LOG_DIR}/{container_name}"),
117+
log_file,
118+
max_log_file_size
119+
.scale_to(BinaryMultiple::Mebi)
120+
.floor()
121+
.value as u32,
122+
CONSOLE_CONVERSION_PATTERN_LOG4J,
123+
log_config,
124+
),
125+
);
126+
}
127+
}
128+
129+
fn add_log4j2_config_if_automatic(
130+
cm_builder: &mut ConfigMapBuilder,
131+
log_config: Option<Cow<ContainerLogConfig>>,
132+
log_config_file: &str,
133+
container_name: impl Display,
134+
log_file: &str,
135+
max_log_file_size: MemoryQuantity,
136+
) {
137+
if let Some(ContainerLogConfig {
138+
choice: Some(ContainerLogConfigChoice::Automatic(log_config)),
139+
}) = log_config.as_deref()
140+
{
141+
cm_builder.add_data(
142+
log_config_file,
143+
product_logging::framework::create_log4j2_config(
144+
&format!("{STACKABLE_LOG_DIR}/{container_name}",),
145+
log_file,
146+
max_log_file_size
147+
.scale_to(BinaryMultiple::Mebi)
148+
.floor()
149+
.value as u32,
150+
CONSOLE_CONVERSION_PATTERN_LOG4J2,
151+
log_config,
152+
),
153+
);
154+
}
155+
}

‎rust/operator-binary/src/resource/configmap.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,12 @@ pub fn build_rolegroup_config_map(
129129
tracing::debug!(?kafka_config, "Applied kafka config");
130130
tracing::debug!(?jvm_sec_props, "Applied JVM config");
131131

132-
extend_role_group_config_map(rolegroup, merged_config, &mut cm_builder);
132+
extend_role_group_config_map(
133+
&resolved_product_image.product_version,
134+
rolegroup,
135+
merged_config,
136+
&mut cm_builder,
137+
);
133138

134139
cm_builder
135140
.build()

‎rust/operator-binary/src/resource/statefulset.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ use crate::{
5151
self, APP_NAME, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME,
5252
LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME,
5353
STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_LISTENER_BOOTSTRAP_DIR,
54-
STACKABLE_LISTENER_BROKER_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR,
54+
STACKABLE_LISTENER_BROKER_DIR,
5555
listener::get_kafka_listener_config,
5656
role::{
5757
AnyConfig, KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer,
@@ -63,7 +63,10 @@ use crate::{
6363
kafka_controller::KAFKA_CONTROLLER_NAME,
6464
kerberos::add_kerberos_pod_config,
6565
operations::graceful_shutdown::add_graceful_shutdown_config,
66-
product_logging::{LOG4J_CONFIG_FILE, MAX_KAFKA_LOG_FILES_SIZE},
66+
product_logging::{
67+
MAX_KAFKA_LOG_FILES_SIZE, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, kafka_log_opts,
68+
kafka_log_opts_env_var,
69+
},
6770
utils::build_recommended_labels,
6871
};
6972

@@ -324,8 +327,8 @@ pub fn build_broker_rolegroup_statefulset(
324327
.context(ConstructJvmArgumentsSnafu)?,
325328
)
326329
.add_env_var(
327-
"KAFKA_LOG4J_OPTS",
328-
format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}"),
330+
kafka_log_opts_env_var(&resolved_product_image.product_version),
331+
kafka_log_opts(&resolved_product_image.product_version),
329332
)
330333
// Needed for the `containerdebug` process to log it's tracing information to.
331334
.add_env_var(
@@ -666,8 +669,8 @@ pub fn build_controller_rolegroup_statefulset(
666669
.context(ConstructJvmArgumentsSnafu)?,
667670
)
668671
.add_env_var(
669-
"KAFKA_LOG4J_OPTS",
670-
format!("-Dlog4j.configuration=file:{STACKABLE_LOG_CONFIG_DIR}/{LOG4J_CONFIG_FILE}"),
672+
kafka_log_opts_env_var(&resolved_product_image.product_version),
673+
kafka_log_opts(&resolved_product_image.product_version),
671674
)
672675
// Needed for the `containerdebug` process to log it's tracing information to.
673676
.add_env_var(

0 commit comments

Comments
 (0)