Enable Log Aggregation for Superset (#326)

# Description

Add log aggregation

Author: siegfriedweber

CHANGELOG.md

@@ -2,14 +2,19 @@
 
 ## [Unreleased]
 
+### Added
+
+- Log aggregation added ([#326]).
+
 ### Changed
 
-- `operator-rs` `0.31.0` -> `0.34.0` ([#322]).
-- Bumped stackable image versions to "23.4.0-rc1" ([#322]).
+- `operator-rs` `0.31.0` -> `0.35.0` ([#322], [#326]).
+- Bumped stackable image versions to "23.4.0-rc2" ([#322], [#326]).
 - Fragmented `SupersetConfig` ([#323]).
 
 [#322]: https://github.com/stackabletech/superset-operator/pull/322
 [#323]: https://github.com/stackabletech/superset-operator/pull/323
+[#326]: https://github.com/stackabletech/superset-operator/pull/326
 
 ## [23.1.0] - 2023-01-23
 

Cargo.lock

@@ -6,7 +6,7 @@ metadata:
 spec:
   image:
     productVersion: 1.5.1
-    stackableVersion: 23.4.0-rc1
+    stackableVersion: 23.4.0-rc2
   credentialsSecret: simple-superset-credentials
   loadExamplesOnInit: true
   nodes:

deploy/helm/superset-operator/crds/crds.yaml

@@ -21,7 +21,7 @@ metadata:
 spec:
   image:
     productVersion: 1.5.1
-    stackableVersion: 23.4.0-rc1
+    stackableVersion: 23.4.0-rc2
   [...]
   authenticationConfig:
     authenticationClass: ldap    # <1>
@@ -101,6 +101,27 @@ image::superset-databases.png[Superset databases showing the connected Druid clu
 The managed Superset instances are automatically configured to export Prometheus metrics. See
 xref:home:operators:monitoring.adoc[] for more details.
 
+== Log aggregation
+
+The logs can be forwarded to a Vector log aggregator by providing a discovery
+ConfigMap for the aggregator and by enabling the log agent:
+
+[source,yaml]
+----
+spec:
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: true
+  databaseInitialization:
+    logging:
+      enableVectorAgent: true
+----
+
+Further information on how to configure logging, can be found in
+xref:home:concepts:logging.adoc[].
+
 == Configuration & Environment Overrides
 
 The cluster definition also supports overriding configuration properties and environment variables,

docs/modules/superset/examples/getting_started/superset.yaml

@@ -150,7 +150,7 @@ metadata:
 spec:
   image:
     productVersion: 1.5.1
-    stackableVersion: 23.4.0-rc1
+    stackableVersion: 23.4.0-rc2
   credentialsSecret: superset-with-ldap-server-veri-tls-credentials
   nodes:
     roleGroups:

docs/modules/superset/pages/usage.adoc

@@ -11,7 +11,7 @@ publish = false
 [dependencies]
 serde = "1.0"
 serde_json = "1.0"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.34.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.35.0" }
 strum = { version = "0.24", features = ["derive"] }
 snafu = "0.7"
 tracing = "0.1"

examples/superset-with-ldap.yaml

@@ -14,21 +14,23 @@ use stackable_operator::{
     kube::{runtime::reflector::ObjectRef, CustomResource},
     product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType},
     product_config_utils::{ConfigError, Configuration},
+    product_logging::{self, spec::Logging},
     role_utils::{Role, RoleGroupRef},
     schemars::{self, JsonSchema},
 };
-use std::{collections::BTreeMap, num::ParseIntError};
+use std::collections::BTreeMap;
 use strum::{Display, EnumIter, EnumString, IntoEnumIterator};
 
 pub const APP_NAME: &str = "superset";
-
+pub const CONFIG_DIR: &str = "/stackable/config";
+pub const LOG_CONFIG_DIR: &str = "/stackable/log_config";
+pub const LOG_DIR: &str = "/stackable/log";
 pub const PYTHONPATH: &str = "/stackable/app/pythonpath";
 pub const SUPERSET_CONFIG_FILENAME: &str = "superset_config.py";
+pub const LOG_VOLUME_SIZE_IN_MIB: u32 = 10;
 
 #[derive(Debug, Snafu)]
 pub enum Error {
-    #[snafu(display("invalid int config value"))]
-    InvalidIntConfigValue { source: ParseIntError },
     #[snafu(display("unknown Superset role found {role}. Should be one of {roles:?}"))]
     UnknownSupersetRole { role: String, roles: Vec<String> },
     #[snafu(display("fragment validation failure"))]
@@ -44,6 +46,7 @@ pub enum SupersetConfigOptions {
     RowLimit,
     MapboxApiKey,
     SupersetWebserverTimeout,
+    LoggingConfigurator,
     AuthType,
     AuthUserRegistration,
     AuthUserRegistrationRole,
@@ -89,6 +92,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions {
             SupersetConfigOptions::RowLimit => PythonType::IntLiteral,
             SupersetConfigOptions::MapboxApiKey => PythonType::Expression,
             SupersetConfigOptions::SupersetWebserverTimeout => PythonType::IntLiteral,
+            SupersetConfigOptions::LoggingConfigurator => PythonType::Expression,
             SupersetConfigOptions::AuthType => PythonType::Expression,
             SupersetConfigOptions::AuthUserRegistration => PythonType::BoolLiteral,
             SupersetConfigOptions::AuthUserRegistrationRole => PythonType::StringLiteral,
@@ -136,6 +140,10 @@ pub struct SupersetClusterSpec {
     pub stopped: Option<bool>,
     /// The Superset image to use
     pub image: ProductImage,
+    /// Name of the Vector aggregator discovery ConfigMap.
+    /// It must contain the key `ADDRESS` with the address of the Vector aggregator.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vector_aggregator_config_map_name: Option<String>,
     pub credentials_secret: String,
     pub mapbox_secret: Option<String>,
     #[serde(default)]
@@ -144,6 +152,8 @@ pub struct SupersetClusterSpec {
     pub authentication_config: Option<SupersetClusterAuthenticationConfig>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub nodes: Option<Role<SupersetConfigFragment>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub database_initialization: Option<supersetdb::SupersetDbConfigFragment>,
     /// Specify the type of the created kubernetes service.
     /// This attribute will be removed in a future release when listener-operator is finished.
     /// Use with caution.
@@ -258,6 +268,26 @@ pub enum SupersetRole {
 )]
 pub struct SupersetStorageConfig {}
 
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Display,
+    Eq,
+    EnumIter,
+    JsonSchema,
+    Ord,
+    PartialEq,
+    PartialOrd,
+    Serialize,
+)]
+#[serde(rename_all = "kebab-case")]
+#[strum(serialize_all = "kebab-case")]
+pub enum Container {
+    Superset,
+    Vector,
+}
+
 #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)]
 #[fragment_attrs(
     derive(
@@ -284,6 +314,8 @@ pub struct SupersetConfig {
     /// CPU and memory limits for Superset pods
     #[fragment_attrs(serde(default))]
     pub resources: Resources<SupersetStorageConfig, NoRuntimeLimits>,
+    #[fragment_attrs(serde(default))]
+    pub logging: Logging<Container>,
 }
 
 impl SupersetConfig {
@@ -303,6 +335,7 @@ impl SupersetConfig {
                 },
                 storage: SupersetStorageConfigFragment {},
             },
+            logging: product_logging::spec::default_logging(),
             ..Default::default()
         }
     }

rust/crd/Cargo.toml

@@ -1,27 +1,77 @@
 use crate::{SupersetCluster, APP_NAME};
 use serde::{Deserialize, Serialize};
-use snafu::Snafu;
-use stackable_operator::builder::ObjectMetaBuilder;
-use stackable_operator::commons::product_image_selection::{ProductImage, ResolvedProductImage};
-use stackable_operator::k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
-use stackable_operator::k8s_openapi::chrono::Utc;
-use stackable_operator::kube::CustomResource;
-use stackable_operator::kube::ResourceExt;
-use stackable_operator::labels::{self, APP_VERSION_LABEL};
-use stackable_operator::schemars::{self, JsonSchema};
+use snafu::{ResultExt, Snafu};
+use stackable_operator::{
+    builder::ObjectMetaBuilder,
+    commons::product_image_selection::{ProductImage, ResolvedProductImage},
+    config::{
+        fragment::{self, Fragment, ValidationError},
+        merge::Merge,
+    },
+    k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, chrono::Utc},
+    kube::CustomResource,
+    kube::ResourceExt,
+    labels::{self, APP_VERSION_LABEL},
+    product_logging::{self, spec::Logging},
+    schemars::{self, JsonSchema},
+};
+use strum::{Display, EnumIter};
 
 #[derive(Snafu, Debug)]
 #[allow(clippy::enum_variant_names)]
 pub enum Error {
-    #[snafu(display("object is missing metadata to build owner reference"))]
-    ObjectMissingMetadataForOwnerRef {
-        source: stackable_operator::error::Error,
-    },
-    #[snafu(display("failed to retrieve superset version"))]
-    NoSupersetVersion,
+    #[snafu(display("fragment validation failure"))]
+    FragmentValidationFailure { source: ValidationError },
 }
 type Result<T, E = Error> = std::result::Result<T, E>;
 
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Display,
+    Eq,
+    EnumIter,
+    JsonSchema,
+    Ord,
+    PartialEq,
+    PartialOrd,
+    Serialize,
+)]
+#[serde(rename_all = "kebab-case")]
+#[strum(serialize_all = "kebab-case")]
+pub enum InitDbContainer {
+    SupersetInitDb,
+    Vector,
+}
+
+#[derive(Clone, Debug, Default, Eq, Fragment, JsonSchema, PartialEq)]
+#[fragment_attrs(
+    derive(
+        Clone,
+        Debug,
+        Default,
+        Deserialize,
+        Merge,
+        JsonSchema,
+        PartialEq,
+        Serialize
+    ),
+    serde(rename_all = "camelCase")
+)]
+pub struct SupersetDbConfig {
+    #[fragment_attrs(serde(default))]
+    pub logging: Logging<InitDbContainer>,
+}
+
+impl SupersetDbConfig {
+    fn default_config() -> SupersetDbConfigFragment {
+        SupersetDbConfigFragment {
+            logging: product_logging::spec::default_logging(),
+        }
+    }
+}
+
 #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
 #[kube(
     group = "superset.stackable.tech",
@@ -42,6 +92,9 @@ pub struct SupersetDBSpec {
     pub image: ProductImage,
     pub credentials_secret: String,
     pub load_examples: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vector_aggregator_config_map_name: Option<String>,
+    pub config: SupersetDbConfigFragment,
 }
 
 impl SupersetDB {
@@ -66,6 +119,18 @@ impl SupersetDB {
                 image: superset.spec.image.clone(),
                 credentials_secret: superset.spec.credentials_secret.clone(),
                 load_examples: superset.spec.load_examples_on_init.unwrap_or_default(),
+                vector_aggregator_config_map_name: superset
+                    .spec
+                    .vector_aggregator_config_map_name
+                    .clone(),
+                config: SupersetDbConfigFragment {
+                    logging: superset
+                        .spec
+                        .database_initialization
+                        .clone()
+                        .unwrap_or_default()
+                        .logging,
+                },
             },
             status: None,
         })
@@ -74,6 +139,13 @@ impl SupersetDB {
     pub fn job_name(&self) -> String {
         self.name_unchecked()
     }
+
+    pub fn merged_config(&self) -> Result<SupersetDbConfig, Error> {
+        let defaults = SupersetDbConfig::default_config();
+        let mut config = self.spec.config.to_owned();
+        config.merge(&defaults);
+        fragment::validate(config).context(FragmentValidationFailureSnafu)
+    }
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)]

rust/crd/src/lib.rs

@@ -10,19 +10,19 @@ publish = false
 
 [dependencies]
 anyhow = "1.0"
-clap = "4.0"
+clap = "4.1"
 fnv = "1.0"
 futures = { version = "0.3", features = ["compat"] }
 indoc = "1.0"
 serde = "1.0"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.34.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.35.0" }
 stackable-superset-crd = { path = "../crd" }
 strum = { version = "0.24", features = ["derive"] }
-tokio = { version = "1.23", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] }
 tracing = "0.1"
 
 [build-dependencies]
 built = { version =  "0.5", features = ["chrono", "git2"] }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.34.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.35.0" }
 stackable-superset-crd = { path = "../crd" }