Logging (#294)

# Description

Add log aggregation

Author: siegfriedweber

CHANGELOG.md

@@ -2,11 +2,15 @@
 
 ## [Unreleased]
 
+### Added
+
+- Log aggregation added ([#294]).
+
 ### Changed
 
 - [BREAKING] Use Product image selection instead of version. `spec.version` has been replaced by `spec.image` ([#282]).
 - Updated stackable image versions ([#275]).
-- `operator-rs` `0.24.0` -> `0.30.1` ([#277], [#293]).
+- `operator-rs` `0.24.0` -> `0.30.2` ([#277], [#293], [#294]).
 - Set runAsGroup to 1000 rather than 0 ([#283]).
 - Fixed: `selector` in role groups now works. It was not working before ([#293])
 
@@ -15,6 +19,7 @@
 [#282]: https://github.com/stackabletech/hbase-operator/pull/282
 [#283]: https://github.com/stackabletech/hbase-operator/pull/283
 [#293]: https://github.com/stackabletech/hbase-operator/pull/293
+[#294]: https://github.com/stackabletech/hbase-operator/pull/294
 
 ## [0.5.0] - 2022-11-07
 

Cargo.lock

@@ -19,7 +19,7 @@ metadata:
 spec:
   image:
     productVersion: 2.4.12
-    stackableVersion: 0.3.0
+    stackableVersion: 0.4.0
   hdfsConfigMapName: simple-hdfs-namenode-default
   zookeeperConfigMapName: simple-hbase-znode
   config:
@@ -63,6 +63,32 @@ It should generally be safe to simply use the latest image version that is avail
 The managed HBase instances are automatically configured to export Prometheus metrics. See
 xref:home:operators:monitoring.adoc[] for more details.
 
+== Log aggregation
+
+The logs can be forwarded to a Vector log aggregator by providing a discovery
+ConfigMap for the aggregator and by enabling the log agent:
+
+[source,yaml]
+----
+spec:
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+  masters:
+    config:
+      logging:
+        enableVectorAgent: true
+  regionServers:
+    config:
+      logging:
+        enableVectorAgent: true
+  restServers:
+    config:
+      logging:
+        enableVectorAgent: true
+----
+
+Further information on how to configure logging, can be found in
+xref:home:concepts:logging.adoc[].
+
 == Configuration Overrides
 
 The cluster definition also supports overriding configuration properties and environment variables, either per role or per role group, where the more specific override (role group) has precedence over the less specific one (role).

deploy/crd/hbasecluster.crd.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   image:
     productVersion: 2.4.12
-    stackableVersion: 0.3.0
+    stackableVersion: 0.4.0
   hdfsConfigMapName: simple-hdfs
   zookeeperConfigMapName: simple-znode
   config:

deploy/helm/hbase-operator/crds/crds.yaml

@@ -9,7 +9,7 @@ version = "0.6.0-nightly"
 publish = false
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" }
 
 serde = "1.0"
 serde_json = "1.0"

deploy/manifests/crds.yaml

@@ -12,6 +12,7 @@ use stackable_operator::{
     k8s_openapi::apimachinery::pkg::api::resource::Quantity,
     kube::{runtime::reflector::ObjectRef, CustomResource},
     product_config_utils::{ConfigError, Configuration},
+    product_logging::{self, spec::Logging},
     role_utils::{Role, RoleGroup, RoleGroupRef},
     schemars::{self, JsonSchema},
 };
@@ -86,14 +87,18 @@ pub struct HbaseClusterSpec {
     pub zookeeper_config_map_name: String,
     /// HDFS cluster connection details from discovery config map
     pub hdfs_config_map_name: String,
+    /// Name of the Vector aggregator discovery ConfigMap.
+    /// It must contain the key `ADDRESS` with the address of the Vector aggregator.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vector_aggregator_config_map_name: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub config: Option<HbaseConfig>,
+    pub config: Option<HbaseConfigFragment>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub masters: Option<Role<HbaseConfig>>,
+    pub masters: Option<Role<HbaseConfigFragment>>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub region_servers: Option<Role<HbaseConfig>>,
+    pub region_servers: Option<Role<HbaseConfigFragment>>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub rest_servers: Option<Role<HbaseConfig>>,
+    pub rest_servers: Option<Role<HbaseConfigFragment>>,
 }
 
 #[derive(
@@ -161,33 +166,72 @@ impl HbaseRole {
 )]
 pub struct HbaseStorageConfig {}
 
-#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Display,
+    Eq,
+    EnumIter,
+    JsonSchema,
+    Ord,
+    PartialEq,
+    PartialOrd,
+    Serialize,
+)]
 #[serde(rename_all = "camelCase")]
+pub enum Container {
+    Hbase,
+    Vector,
+}
+
+#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)]
+#[fragment_attrs(
+    derive(
+        Clone,
+        Debug,
+        Default,
+        Deserialize,
+        Merge,
+        JsonSchema,
+        PartialEq,
+        Serialize
+    ),
+    serde(rename_all = "camelCase")
+)]
 pub struct HbaseConfig {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub hbase_rootdir: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub hbase_opts: Option<String>,
-    pub resources: Option<ResourcesFragment<HbaseStorageConfig, NoRuntimeLimits>>,
+    #[fragment_attrs(serde(default))]
+    pub resources: Resources<HbaseStorageConfig, NoRuntimeLimits>,
+    #[fragment_attrs(serde(default))]
+    pub logging: Logging<Container>,
 }
 
 impl HbaseConfig {
-    fn default_resources() -> ResourcesFragment<HbaseStorageConfig, NoRuntimeLimits> {
-        ResourcesFragment {
-            cpu: CpuLimitsFragment {
-                min: Some(Quantity("200m".to_owned())),
-                max: Some(Quantity("4".to_owned())),
-            },
-            memory: MemoryLimitsFragment {
-                limit: Some(Quantity("2Gi".to_owned())),
-                runtime_limits: NoRuntimeLimitsFragment {},
+    fn default_config() -> HbaseConfigFragment {
+        HbaseConfigFragment {
+            hbase_rootdir: None,
+            hbase_opts: None,
+            resources: ResourcesFragment {
+                cpu: CpuLimitsFragment {
+                    min: Some(Quantity("200m".to_owned())),
+                    max: Some(Quantity("4".to_owned())),
+                },
+                memory: MemoryLimitsFragment {
+                    limit: Some(Quantity("2Gi".to_owned())),
+                    runtime_limits: NoRuntimeLimitsFragment {},
+                },
+                storage: HbaseStorageConfigFragment {},
             },
-            storage: HbaseStorageConfigFragment {},
+            logging: product_logging::spec::default_logging(),
         }
     }
 }
 
-impl Configuration for HbaseConfig {
+impl Configuration for HbaseConfigFragment {
     type Configurable = HbaseCluster;
 
     fn compute_env(
@@ -277,7 +321,7 @@ impl HbaseCluster {
         }
     }
 
-    pub fn get_role(&self, role: &HbaseRole) -> Option<&Role<HbaseConfig>> {
+    pub fn get_role(&self, role: &HbaseRole) -> Option<&Role<HbaseConfigFragment>> {
         match role {
             HbaseRole::Master => self.spec.masters.as_ref(),
             HbaseRole::RegionServer => self.spec.region_servers.as_ref(),
@@ -289,7 +333,7 @@ impl HbaseCluster {
     pub fn get_role_group(
         &self,
         rolegroup_ref: &RoleGroupRef<HbaseCluster>,
-    ) -> Result<&RoleGroup<HbaseConfig>, Error> {
+    ) -> Result<&RoleGroup<HbaseConfigFragment>, Error> {
         let role_variant =
             HbaseRole::from_str(&rolegroup_ref.role).with_context(|_| InvalidRoleSnafu {
                 role: rolegroup_ref.role.to_owned(),
@@ -316,27 +360,26 @@ impl HbaseCluster {
     }
 
     /// Retrieve and merge resource configs for role and role groups
-    pub fn resolve_resource_config_for_role_and_rolegroup(
+    pub fn merged_config(
         &self,
         role: &HbaseRole,
         rolegroup_ref: &RoleGroupRef<HbaseCluster>,
-    ) -> Result<Resources<HbaseStorageConfig, NoRuntimeLimits>, Error> {
+    ) -> Result<HbaseConfig, Error> {
         // Initialize the result with all default values as baseline
-        let conf_defaults = HbaseConfig::default_resources();
+        let conf_defaults = HbaseConfig::default_config();
 
         let role = self.get_role(role).context(MissingHbaseRoleSnafu {
             role: role.to_string(),
         })?;
 
         // Retrieve role resource config
-        let mut conf_role: ResourcesFragment<HbaseStorageConfig, NoRuntimeLimits> =
-            role.config.config.resources.clone().unwrap_or_default();
+        let mut conf_role = role.config.config.to_owned();
 
         // Retrieve rolegroup specific resource config
-        let mut conf_rolegroup: ResourcesFragment<HbaseStorageConfig, NoRuntimeLimits> = role
+        let mut conf_rolegroup = role
             .role_groups
             .get(&rolegroup_ref.role_group)
-            .and_then(|rg| rg.config.config.resources.clone())
+            .map(|rg| rg.config.config.clone())
             .unwrap_or_default();
 
         // Merge more specific configs into default config
@@ -347,7 +390,7 @@ impl HbaseCluster {
         conf_role.merge(&conf_defaults);
         conf_rolegroup.merge(&conf_role);
 
-        tracing::debug!("Merged resource config: {:?}", conf_rolegroup);
+        tracing::debug!("Merged config: {:?}", conf_rolegroup);
         fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu)
     }
 }

docs/modules/ROOT/pages/usage.adoc

@@ -16,12 +16,12 @@ futures = { version = "0.3", features = ["compat"] }
 serde = "1.0"
 snafu = "0.7"
 stackable-hbase-crd = { path = "../crd" }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" }
 strum = { version = "0.24", features = ["derive"] }
 tokio = { version = "1.23", features = ["macros", "rt-multi-thread"] }
 tracing = "0.1"
 
 [build-dependencies]
 built = { version =  "0.5", features = ["chrono", "git2"] }
 stackable-hbase-crd = { path = "../crd" }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" }