refactor: Add ValidatedHbaseConfig, adapt to v2 logging & add vector tests

Author: maltesander

rust/operator-binary/src/controller/build/properties/logging.rs

@@ -0,0 +1,68 @@
+//! Renders the logging config files (`log4j2.properties` and the Vector agent config)
+//! assembled into the rolegroup `ConfigMap`.
+
+pub use stackable_operator::v2::product_logging::framework::STACKABLE_LOG_DIR;
+use stackable_operator::{
+    memory::{BinaryMultiple, MemoryQuantity},
+    product_logging::{self, spec::AutomaticContainerLogConfig},
+    v2::product_logging::framework::ValidatedContainerLogConfigChoice,
+};
+
+pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity {
+    value: 10.0,
+    unit: BinaryMultiple::Mebi,
+};
+
+const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n";
+const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml";
+
+/// The Vector agent configuration (`vector.yaml`).
+///
+/// This is a static, env-var-parameterized file (mirroring the hive- and opensearch-operators).
+/// It is validated by the `test-vector.sh` / `vector-test.yaml` harness next to this file.
+const VECTOR_CONFIG: &str = include_str!("vector.yaml");
+
+/// Returns the Vector agent config (`vector.yaml`) content.
+pub fn vector_config_file_content() -> String {
+    VECTOR_CONFIG.to_owned()
+}
+
+/// Renders `log4j2.properties` for the HBase container.
+///
+/// Returns `None` when the HBase container uses a custom log ConfigMap instead of the operator's
+/// automatic logging configuration, in which case no `log4j2.properties` should be added to the
+/// rolegroup `ConfigMap`.
+pub fn build_log4j2(hbase_container: &ValidatedContainerLogConfigChoice) -> Option<String> {
+    match hbase_container {
+        ValidatedContainerLogConfigChoice::Automatic(log_config) => Some(log4j_config(log_config)),
+        ValidatedContainerLogConfigChoice::Custom(_) => None,
+    }
+}
+
+fn log4j_config(log_config: &AutomaticContainerLogConfig) -> String {
+    product_logging::framework::create_log4j2_config(
+        &format!("{STACKABLE_LOG_DIR}/hbase"),
+        HBASE_LOG4J2_FILE,
+        MAX_HBASE_LOG_FILES_SIZE
+            .scale_to(BinaryMultiple::Mebi)
+            .floor()
+            .value as u32,
+        CONSOLE_CONVERSION_PATTERN,
+        log_config,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_vector_config_file_content() {
+        let content = vector_config_file_content();
+        assert!(!content.is_empty());
+        // A kept source must be present ...
+        assert!(content.contains("files_log4j2"));
+        // ... while a product-specific source we don't emit must not.
+        assert!(!content.contains("files_tracing_rs"));
+    }
+}

rust/operator-binary/src/controller/build/properties/logging/mod.rs

@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+
+DATA_DIR=/stackable/log/_vector-state \
+LOG_DIR=/stackable/log \
+NAMESPACE=default \
+CLUSTER_NAME=hbase \
+ROLE_NAME=regionserver \
+ROLE_GROUP_NAME=default \
+VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \
+VECTOR_FILE_LOG_LEVEL=info \
+vector test vector.yaml vector-test.yaml

rust/operator-binary/src/controller/build/properties/logging/test-vector.sh

@@ -0,0 +1,162 @@
+# Run tests with `./test-vector.sh`
+#
+# A downside of these test cases is that they compare the whole event and that the message can
+# contain source code positions in vector.yaml, e.g. "function call error for \"parse_xml\" at
+# (584:643)". Please adapt the tests if you change VRL code in vector.yaml.
+---
+tests:
+  - name: Test stdout log entry
+    inputs:
+      - type: log
+        insert_at: processed_files_stdout
+        log_fields:
+          file: /stackable/log/hbase/hbase.stdout.log
+          message: Starting HRegionServer
+          pod: hbase-regionserver-default-0
+          source_type: file
+          timestamp: 2025-10-02T09:27:28.582Z
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "hbase",
+                "container": "hbase",
+                "file": "hbase.stdout.log",
+                "level": "INFO",
+                "logger": "ROOT",
+                "message": "Starting HRegionServer",
+                "namespace": "default",
+                "pod": "hbase-regionserver-default-0",
+                "role": "regionserver",
+                "roleGroup": "default",
+                "timestamp": "2025-10-02T09:27:28.582Z"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test stderr log entry
+    inputs:
+      - type: log
+        insert_at: processed_files_stderr
+        log_fields:
+          file: /stackable/log/hbase/hbase.stderr.log
+          message: "Exception in thread \"main\""
+          pod: hbase-regionserver-default-0
+          source_type: file
+          timestamp: 2025-10-02T09:27:28.582Z
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "hbase",
+                "container": "hbase",
+                "file": "hbase.stderr.log",
+                "level": "ERROR",
+                "logger": "ROOT",
+                "message": "Exception in thread \"main\"",
+                "namespace": "default",
+                "pod": "hbase-regionserver-default-0",
+                "role": "regionserver",
+                "roleGroup": "default",
+                "timestamp": "2025-10-02T09:27:28.582Z"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test log4j2 XML log entry without stacktrace
+    inputs:
+      - type: log
+        insert_at: processed_files_log4j2
+        log_fields:
+          file: /stackable/log/hbase/hbase.log4j2.xml
+          message: >
+            <Event xmlns="http://logging.apache.org/log4j/2.0/events"
+            timeMillis="1759397248582" level="INFO"
+            loggerName="org.apache.hadoop.hbase.regionserver.HRegionServer"
+            thread="main"><Message>Serving as
+            regionserver...</Message></Event>
+          pod: hbase-regionserver-default-0
+          source_type: file
+          timestamp: 2025-10-02T09:27:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "hbase",
+                "container": "hbase",
+                "file": "hbase.log4j2.xml",
+                "level": "INFO",
+                "logger": "org.apache.hadoop.hbase.regionserver.HRegionServer",
+                "message": "Serving as regionserver...",
+                "namespace": "default",
+                "pod": "hbase-regionserver-default-0",
+                "role": "regionserver",
+                "roleGroup": "default",
+                "timestamp": t'2025-10-02T09:27:28.582Z'
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test Vector internal logs
+    inputs:
+      - type: log
+        insert_at: filtered_logs_vector
+        log_fields:
+          arch: x86_64
+          message: Vector has started.
+          metadata:
+            kind: event
+            level: INFO
+            module_path: vector::internal_events::process
+            target: vector
+          pid: 14
+          pod: hbase-regionserver-default-0
+          source_type: internal_logs
+          timestamp: 2025-10-02T09:46:14.479381097Z
+          version: 0.49.0
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "arch": "x86_64",
+                "cluster": "hbase",
+                "container": "vector",
+                "level": "INFO",
+                "logger": "vector::internal_events::process",
+                "message": "Vector has started.",
+                "namespace": "default",
+                "pod": "hbase-regionserver-default-0",
+                "role": "regionserver",
+                "roleGroup": "default",
+                "timestamp": "2025-10-02T09:46:14.479381097Z",
+                "version": "0.49.0"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test Vector internal log level filtering - INFO passes
+    inputs:
+      - type: log
+        insert_at: filtered_logs_vector
+        log_fields:
+          metadata:
+            level: INFO
+    outputs:
+      - extract_from: filtered_logs_vector
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!("INFO", .metadata.level)
+  - name: Test Vector internal log level filtering - DEBUG dropped
+    inputs:
+      - type: log
+        insert_at: filtered_logs_vector
+        log_fields:
+          metadata:
+            level: DEBUG
+    no_outputs_from:
+      - filtered_logs_vector