Skip to content

Commit f1a8361

Browse files
maltesandermaltesander
committed
refactor: move to logging v2, add vector tests
1 parent 1a96afe commit f1a8361

17 files changed

Lines changed: 555 additions & 544 deletions

File tree

‎rust/operator-binary/src/controller/build/config_map.rs‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ use crate::{
1515
build::{
1616
git_sync,
1717
properties::{
18-
ConfigFileName, authorizers, bootstrap_conf, logging, login_identity_providers,
19-
nifi_properties, security_properties, state_management_xml,
18+
ConfigFileName, authorizers, bootstrap_conf, login_identity_providers,
19+
nifi_properties, product_logging, security_properties, state_management_xml,
2020
},
2121
proxy_hosts,
2222
},
@@ -69,14 +69,10 @@ type Result<T, E = Error> = std::result::Result<T, E>;
6969
/// resolved cluster configuration.
7070
///
7171
/// All NiFi configuration is sourced from `cluster`.
72-
///
73-
/// `vector_config` is the Vector agent config (`vector.yaml`) built by the caller (where a
74-
/// `RoleGroupRef` is available); it is `None` when the Vector agent is disabled.
7572
pub fn build_rolegroup_config_map(
7673
cluster: &ValidatedCluster,
7774
role_group_name: &RoleGroupName,
7875
cluster_info: &KubernetesClusterInfo,
79-
vector_config: Option<String>,
8076
) -> Result<ConfigMap> {
8177
tracing::debug!("building rolegroup ConfigMap");
8278

@@ -142,12 +138,15 @@ pub fn build_rolegroup_config_map(
142138
})?,
143139
);
144140

145-
if let Some(logback_config) = logging::build_logback_config(&rg.config.logging) {
141+
if let Some(logback_config) = product_logging::build_logback_config(&rg.config.logging) {
146142
cm_builder.add_data(ConfigFileName::Logback.to_string(), logback_config);
147143
}
148144

149-
if let Some(vector_config) = vector_config {
150-
cm_builder.add_data(VECTOR_CONFIG_FILE, vector_config);
145+
if rg.config.logging.enable_vector_agent {
146+
cm_builder.add_data(
147+
VECTOR_CONFIG_FILE,
148+
product_logging::vector_config_file_content(),
149+
);
151150
}
152151

153152
cm_builder

‎rust/operator-binary/src/controller/build/properties.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ use std::{collections::BTreeMap, fmt::Write};
88

99
pub mod authorizers;
1010
pub mod bootstrap_conf;
11-
pub mod logging;
1211
pub mod login_identity_providers;
1312
pub mod nifi_properties;
13+
pub mod product_logging;
1414
pub mod security_properties;
1515
pub mod state_management_xml;
1616

@@ -130,7 +130,7 @@ pub(crate) mod test_support {
130130
let nifi: v1alpha1::NifiCluster =
131131
serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML");
132132

133-
let role_group_configs = build_role_group_configs(&nifi)
133+
let role_group_configs = build_role_group_configs(&nifi, &None)
134134
.expect("role group configs should merge for minimal fixture");
135135

136136
let image = ResolvedProductImage {

‎rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ mod tests {
6464
serde_yaml::from_str(nifi_cluster).expect("illegal test input");
6565

6666
let role_group_configs =
67-
build_role_group_configs(&nifi).expect("failed to build role group configs");
67+
build_role_group_configs(&nifi, &None).expect("failed to build role group configs");
6868
let rg = role_group_configs
6969
.get(&NifiRole::Node)
7070
.and_then(|groups| {

‎rust/operator-binary/src/controller/build/properties/nifi_properties.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,7 @@ mod tests {
674674
"#;
675675
let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(yaml).expect("invalid test YAML");
676676
let mut role_group_configs =
677-
build_role_group_configs(&nifi).expect("failed to build role group configs");
677+
build_role_group_configs(&nifi, &None).expect("failed to build role group configs");
678678
let default_rg_name = "default"
679679
.parse::<RoleGroupName>()
680680
.expect("valid role-group name");

rust/operator-binary/src/controller/build/properties/logging.rs renamed to rust/operator-binary/src/controller/build/properties/product_logging/mod.rs

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,26 @@ use stackable_operator::{
66
self,
77
spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging},
88
},
9-
role_utils::RoleGroupRef,
109
};
1110

12-
use crate::crd::{Container, MAX_NIFI_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1};
11+
use crate::crd::{Container, MAX_NIFI_LOG_FILES_SIZE, STACKABLE_LOG_DIR};
1312

1413
pub const NIFI_LOG_FILE: &str = "nifi.log4j.xml";
1514

15+
/// The Vector agent configuration (`vector.yaml`).
16+
///
17+
/// Embedded statically and shipped in the rolegroup `ConfigMap`; the per-rolegroup values
18+
/// (namespace, cluster/role/role-group, aggregator address, log levels) are injected as
19+
/// environment variables by the Vector container (see
20+
/// [`stackable_operator::v2::product_logging::framework::vector_container`]) and substituted by
21+
/// Vector at runtime.
22+
const VECTOR_CONFIG: &str = include_str!("vector.yaml");
23+
24+
/// Returns the Vector agent config (`vector.yaml`) content.
25+
pub fn vector_config_file_content() -> String {
26+
VECTOR_CONFIG.to_owned()
27+
}
28+
1629
const CONSOLE_CONVERSION_PATTERN: &str = "%date %level [%thread] %logger{40} %msg%n";
1730
// This is required to remove double entries in the nifi.log4j.xml as well as nested
1831
// console output like: "<timestamp> <loglevel> ... <timestamp> <loglevel> ...
@@ -59,28 +72,15 @@ pub fn build_logback_config(logging: &Logging<Container>) -> Option<String> {
5972
))
6073
}
6174

62-
/// Renders the Vector agent config (`vector.yaml`).
63-
///
64-
/// Returns `None` when the Vector agent is disabled for this role group.
65-
pub fn build_vector_config(
66-
rolegroup: &RoleGroupRef<v1alpha1::NifiCluster>,
67-
logging: &Logging<Container>,
68-
) -> Option<String> {
69-
if !logging.enable_vector_agent {
70-
return None;
71-
}
72-
73-
let vector_log_config = if let Some(ContainerLogConfig {
74-
choice: Some(ContainerLogConfigChoice::Automatic(log_config)),
75-
}) = logging.containers.get(&Container::Vector)
76-
{
77-
Some(log_config)
78-
} else {
79-
None
80-
};
75+
#[cfg(test)]
76+
mod tests {
77+
use super::*;
8178

82-
Some(product_logging::framework::create_vector_config(
83-
rolegroup,
84-
vector_log_config,
85-
))
79+
#[test]
80+
fn test_vector_config_file_content() {
81+
let content = vector_config_file_content();
82+
assert!(!content.is_empty());
83+
// NiFi logs via logback's `XMLLayout` (log4j XML), so the `files_log4j` source
84+
assert!(content.contains("files_log4j"));
85+
}
8686
}

‎rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env sh
2+
3+
DATA_DIR=/stackable/log/_vector-state \
4+
LOG_DIR=/stackable/log \
5+
NAMESPACE=default \
6+
CLUSTER_NAME=nifi \
7+
ROLE_NAME=node \
8+
ROLE_GROUP_NAME=default \
9+
VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \
10+
VECTOR_FILE_LOG_LEVEL=info \
11+
vector test vector.yaml vector-test.yaml

‎rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
# Run tests with `./test-vector.sh`
2+
#
3+
# A downside of these test cases is that they compare the whole event and that the message can
4+
# contain source code positions in vector.yaml, e.g. "function call error for \"parse_xml\" at
5+
# (584:643)". Please adapt the tests if you change VRL code in vector.yaml.
6+
---
7+
tests:
8+
- name: Test stdout log entry
9+
inputs:
10+
- type: log
11+
insert_at: processed_files_stdout
12+
log_fields:
13+
file: /stackable/log/nifi/nifi.stdout.log
14+
message: Starting Apache NiFi
15+
pod: nifi-node-default-0
16+
source_type: file
17+
timestamp: 2025-10-02T09:27:28.582Z
18+
outputs:
19+
- extract_from: extended_logs
20+
conditions:
21+
- type: vrl
22+
source: |
23+
expected_log_event = {
24+
"cluster": "nifi",
25+
"container": "nifi",
26+
"file": "nifi.stdout.log",
27+
"level": "INFO",
28+
"logger": "ROOT",
29+
"message": "Starting Apache NiFi",
30+
"namespace": "default",
31+
"pod": "nifi-node-default-0",
32+
"role": "node",
33+
"roleGroup": "default",
34+
"timestamp": "2025-10-02T09:27:28.582Z"
35+
}
36+
37+
assert_eq!(expected_log_event, .)
38+
- name: Test stderr log entry
39+
inputs:
40+
- type: log
41+
insert_at: processed_files_stderr
42+
log_fields:
43+
file: /stackable/log/nifi/nifi.stderr.log
44+
message: "Exception in thread \"main\""
45+
pod: nifi-node-default-0
46+
source_type: file
47+
timestamp: 2025-10-02T09:27:28.582Z
48+
outputs:
49+
- extract_from: extended_logs
50+
conditions:
51+
- type: vrl
52+
source: |
53+
expected_log_event = {
54+
"cluster": "nifi",
55+
"container": "nifi",
56+
"file": "nifi.stderr.log",
57+
"level": "ERROR",
58+
"logger": "ROOT",
59+
"message": "Exception in thread \"main\"",
60+
"namespace": "default",
61+
"pod": "nifi-node-default-0",
62+
"role": "node",
63+
"roleGroup": "default",
64+
"timestamp": "2025-10-02T09:27:28.582Z"
65+
}
66+
67+
assert_eq!(expected_log_event, .)
68+
- name: Test log4j XML log entry without throwable
69+
inputs:
70+
- type: log
71+
insert_at: processed_files_log4j
72+
log_fields:
73+
file: /stackable/log/nifi/nifi.log4j.xml
74+
message: >
75+
<log4j:event logger="org.apache.nifi.web.server.JettyServer"
76+
timestamp="1704067200000" level="INFO"
77+
thread="main"><log4j:message>NiFi has
78+
started</log4j:message></log4j:event>
79+
pod: nifi-node-default-0
80+
source_type: file
81+
timestamp: 2025-10-02T09:27:29.473487331Z
82+
outputs:
83+
- extract_from: extended_logs
84+
conditions:
85+
- type: vrl
86+
source: |
87+
expected_log_event = {
88+
"cluster": "nifi",
89+
"container": "nifi",
90+
"file": "nifi.log4j.xml",
91+
"level": "INFO",
92+
"logger": "org.apache.nifi.web.server.JettyServer",
93+
"message": "NiFi has started",
94+
"namespace": "default",
95+
"pod": "nifi-node-default-0",
96+
"role": "node",
97+
"roleGroup": "default",
98+
"timestamp": t'2024-01-01T00:00:00Z'
99+
}
100+
101+
assert_eq!(expected_log_event, .)
102+
- name: Test Vector internal logs
103+
inputs:
104+
- type: log
105+
insert_at: filtered_logs_vector
106+
log_fields:
107+
arch: x86_64
108+
message: Vector has started.
109+
metadata:
110+
kind: event
111+
level: INFO
112+
module_path: vector::internal_events::process
113+
target: vector
114+
pid: 14
115+
pod: nifi-node-default-0
116+
source_type: internal_logs
117+
timestamp: 2025-10-02T09:46:14.479381097Z
118+
version: 0.49.0
119+
outputs:
120+
- extract_from: extended_logs
121+
conditions:
122+
- type: vrl
123+
source: |
124+
expected_log_event = {
125+
"arch": "x86_64",
126+
"cluster": "nifi",
127+
"container": "vector",
128+
"level": "INFO",
129+
"logger": "vector::internal_events::process",
130+
"message": "Vector has started.",
131+
"namespace": "default",
132+
"pod": "nifi-node-default-0",
133+
"role": "node",
134+
"roleGroup": "default",
135+
"timestamp": "2025-10-02T09:46:14.479381097Z",
136+
"version": "0.49.0"
137+
}
138+
139+
assert_eq!(expected_log_event, .)
140+
- name: Test Vector internal log level filtering - INFO passes
141+
inputs:
142+
- type: log
143+
insert_at: filtered_logs_vector
144+
log_fields:
145+
metadata:
146+
level: INFO
147+
outputs:
148+
- extract_from: filtered_logs_vector
149+
conditions:
150+
- type: vrl
151+
source: |
152+
assert_eq!("INFO", .metadata.level)
153+
- name: Test Vector internal log level filtering - DEBUG dropped
154+
inputs:
155+
- type: log
156+
insert_at: filtered_logs_vector
157+
log_fields:
158+
metadata:
159+
level: DEBUG
160+
no_outputs_from:
161+
- filtered_logs_vector

0 commit comments

Comments
 (0)