refactor: adapt to v2 logging and add vector test

Author: maltesander

rust/operator-binary/src/controller/build/properties/config_json.rs

@@ -151,7 +151,8 @@ mod tests {
             .values()
             .next()
             .expect("the default role group should exist");
-        let rendered = build(&rg.config, &rg.config_overrides).expect("config.json builds");
+        let rendered =
+            build(&rg.config.config, &rg.config.config_overrides).expect("config.json builds");
         serde_json::from_str(&rendered).expect("config.json should be valid JSON")
     }
 

rust/operator-binary/src/controller/build/properties/logging.rs

@@ -1,7 +1,7 @@
 //! Per-config-file build steps assembled into the rolegroup `ConfigMap`.
 
 pub mod config_json;
-pub mod logging;
+pub mod product_logging;
 pub mod user_info_fetcher;
 
 /// The names of the config files assembled into the rolegroup `ConfigMap`.

rust/operator-binary/src/controller/build/properties/mod.rs

@@ -0,0 +1,50 @@
+//! Renders the Vector agent config (`vector.yaml`) assembled into the rolegroup `ConfigMap`, and
+//! maps log levels for the Stackable Rust sidecars.
+
+use stackable_operator::product_logging::spec::LogLevel;
+
+/// The Vector agent configuration (`vector.yaml`).
+const VECTOR_CONFIG: &str = include_str!("vector.yaml");
+
+/// Returns the Vector agent config (`vector.yaml`) content added to the rolegroup `ConfigMap`.
+pub fn vector_config_file_content() -> String {
+    VECTOR_CONFIG.to_owned()
+}
+
+/// The log level passed to the Stackable Rust sidecars (bundle-builder, user-info-fetcher) via the
+/// `CONSOLE_LOG_LEVEL`/`FILE_LOG_LEVEL` environment variables.
+#[derive(strum::Display)]
+#[strum(serialize_all = "UPPERCASE")]
+pub enum BundleBuilderLogLevel {
+    Trace,
+    Debug,
+    Info,
+    Warn,
+    Error,
+}
+
+impl From<LogLevel> for BundleBuilderLogLevel {
+    fn from(level: LogLevel) -> Self {
+        match level {
+            LogLevel::TRACE => Self::Trace,
+            LogLevel::DEBUG => Self::Debug,
+            LogLevel::INFO => Self::Info,
+            LogLevel::WARN => Self::Warn,
+            LogLevel::ERROR | LogLevel::FATAL | LogLevel::NONE => Self::Error,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn vector_config_file_content_has_opa_sources() {
+        let content = vector_config_file_content();
+        assert!(!content.is_empty());
+        // OPA's own JSON logs and the Rust sidecar (tracing-rs) logs must both be handled.
+        assert!(content.contains("files_opa_json"));
+        assert!(content.contains("files_tracing_rs"));
+    }
+}

rust/operator-binary/src/controller/build/properties/product_logging/mod.rs

@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+
+DATA_DIR=/stackable/log/_vector-state \
+LOG_DIR=/stackable/log \
+NAMESPACE=default \
+CLUSTER_NAME=opa \
+ROLE_NAME=server \
+ROLE_GROUP_NAME=default \
+VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \
+VECTOR_FILE_LOG_LEVEL=info \
+vector test vector.yaml vector-test.yaml

rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh

@@ -0,0 +1,185 @@
+# Run tests with `./test-vector.sh` (requires the `vector` binary).
+#
+# The tests assert individual fields (rather than the whole event) because the `message` field
+# embeds `encode_key_value` output whose ordering is an implementation detail. Please adapt the
+# tests if you change the VRL code in vector.yaml.
+---
+tests:
+  - name: OPA server log (valid)
+    inputs:
+      - type: log
+        insert_at: processed_files_opa_json
+        log_fields:
+          file: /stackable/log/opa/test
+          message: '{"level":"info","msg":"Valid server log event","name":"stackable","plugin":"bundle","time":"2024-01-01T00:00:00Z","logger":"server"}'
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.container, "opa")
+              assert_eq!(.file, "test")
+              assert_eq!(.level, "INFO")
+              assert_eq!(.logger, "server")
+              assert_eq!(.namespace, "default")
+              assert_eq!(.cluster, "opa")
+              assert_eq!(.role, "server")
+              assert_eq!(.roleGroup, "default")
+              assert!(!exists(.errors), "valid event should not carry errors")
+              assert!(contains(string!(.message), "Valid server log event"))
+
+  - name: OPA decision log (valid)
+    inputs:
+      - type: log
+        insert_at: processed_files_opa_json
+        log_fields:
+          file: /stackable/log/opa/test
+          message: '{"level":"info","msg":"Valid decision log event","decision_id":"5b887ec2","time":"2024-01-01T00:00:00Z","logger":"decision"}'
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.container, "opa")
+              assert_eq!(.logger, "decision")
+              assert_eq!(.level, "INFO")
+              assert!(!exists(.errors))
+
+  - name: OPA log with unknown level (invalid)
+    inputs:
+      - type: log
+        insert_at: processed_files_opa_json
+        log_fields:
+          file: /stackable/log/opa/test
+          message: '{"level":"critical","msg":"Invalid log event with an unknown level","time":"2024-01-01T00:00:00Z","logger":"server"}'
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert!(exists(.errors), "unknown level should be recorded as an error")
+              assert_eq!(.level, "INFO")
+
+  - name: Bundle-builder tracing-rs log (valid)
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/bundle-builder/bundle-builder.tracing-rs.json
+          message: '{"timestamp":"2024-01-01T00:00:00.000000Z","level":"INFO","fields":{"message":"Valid log event"},"target":"TestLogger"}'
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.container, "bundle-builder")
+              assert_eq!(.level, "INFO")
+              assert_eq!(.logger, "TestLogger")
+              assert!(!exists(.errors))
+              assert!(contains(string!(.message), "Valid log event"))
+
+  - name: Tracing-rs log without timestamp (invalid)
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/bundle-builder/bundle-builder.tracing-rs.json
+          message: '{"level":"INFO","fields":{"message":"Invalid log event with an unparsable timestamp"},"target":"TestLogger","timestamp":"unparsable timestamp"}'
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert!(exists(.errors), "unparsable timestamp should be recorded as an error")
+
+  - name: Prepare stdout log
+    inputs:
+      - type: log
+        insert_at: processed_files_stdout
+        log_fields:
+          file: /stackable/log/prepare/prepare.stdout.log
+          message: Create dir [/bundles/active]
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.container, "prepare")
+              assert_eq!(.logger, "ROOT")
+              assert_eq!(.level, "INFO")
+
+  - name: Prepare stderr log
+    inputs:
+      - type: log
+        insert_at: processed_files_stderr
+        log_fields:
+          file: /stackable/log/prepare/prepare.stderr.log
+          message: something failed
+          pod: test-opa-server-default-0
+          source_type: file
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.container, "prepare")
+              assert_eq!(.logger, "ROOT")
+              assert_eq!(.level, "ERROR")
+
+  - name: Vector internal log
+    inputs:
+      - type: log
+        insert_at: filtered_logs_vector
+        log_fields:
+          message: Vector has started.
+          metadata:
+            level: INFO
+            module_path: vector::internal_events::process
+          pid: 14
+          pod: test-opa-server-default-0
+          source_type: internal_logs
+    outputs:
+      - extract_from: extended_logs
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.container, "vector")
+              assert_eq!(.level, "INFO")
+              assert_eq!(.logger, "vector::internal_events::process")
+              assert_eq!(.cluster, "opa")
+
+  - name: Vector internal log level filtering - INFO passes
+    inputs:
+      - type: log
+        insert_at: filtered_logs_vector
+        log_fields:
+          metadata:
+            level: INFO
+    outputs:
+      - extract_from: filtered_logs_vector
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!("INFO", .metadata.level)
+
+  - name: Vector internal log level filtering - DEBUG dropped
+    inputs:
+      - type: log
+        insert_at: filtered_logs_vector
+        log_fields:
+          metadata:
+            level: DEBUG
+    no_outputs_from:
+      - filtered_logs_vector