chore: Use internal secret for JWT key

CHANGELOG.md

@@ -9,6 +9,10 @@
 - Added a note on webserver workers to the trouble-shooting section ([#685]).
 - Helm: Allow Pod `priorityClassName` to be configured ([#687]).
 
+### Changed
+
+- Use internal secrets for secret- and jwt-keys ([#686]).
+
 ### Fixed
 
 - Don't panic on invalid authorization config. Previously, a missing OPA ConfigMap would crash the operator ([#667]).
@@ -25,6 +29,7 @@
 [#679]: https://github.com/stackabletech/airflow-operator/pull/679
 [#683]: https://github.com/stackabletech/airflow-operator/pull/683
 [#685]: https://github.com/stackabletech/airflow-operator/pull/685
+[#686]: https://github.com/stackabletech/airflow-operator/pull/686
 [#687]: https://github.com/stackabletech/airflow-operator/pull/687
 
 ## [25.7.0] - 2025-07-23

Cargo.toml

@@ -14,12 +14,14 @@ product-config = { git = "https://github.com/stackabletech/product-config.git",
 stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.95.0" }
 
 anyhow = "1.0"
+base64 = "0.22"
 built = { version = "0.8", features = ["chrono", "git2"] }
 clap = "4.5"
 const_format = "0.2"
 fnv = "1.0"
 futures = { version = "0.3", features = ["compat"] }
 indoc = "2.0"
+rand = "0.9.0"
 rstest = "0.26"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }

docs/modules/airflow/examples/example-airflow-secret.yaml

@@ -10,7 +10,6 @@ stringData:
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.secretKey: thisISaSECRET_1234
   connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
   # Only needed when using celery workers (instead of Kubernetes executors)
   connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow

docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml

@@ -10,7 +10,6 @@ stringData:
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.secretKey: thisISaSECRET_1234
   connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
   # Only needed when using celery workers (instead of Kubernetes executors)
   connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow

docs/modules/airflow/pages/getting_started/first_steps.adoc

@@ -23,9 +23,6 @@ And apply it:
 [source,bash]
 include::example$getting_started/code/getting_started.sh[tag=apply-airflow-credentials]
 
-The `connections.secretKey` is used for securely signing the session cookies and can be used for any other security related needs by extensions.
-It should be a long random string of bytes.
-
 `connections.sqlalchemyDatabaseUri` must contain the connection string to the SQL database storing the Airflow metadata.
 
 `connections.celeryResultBackend` must contain the connection string to the SQL database storing the job metadata (the example above uses the same PostgreSQL database for both).

examples/simple-airflow-cluster-dags-cmap.yaml

@@ -10,7 +10,6 @@ stringData:
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.secretKey: thisISaSECRET_1234
   connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
   # Only needed when using celery workers (instead of Kubernetes executors)
   connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow

examples/simple-airflow-cluster-ldap-insecure-tls.yaml

@@ -138,7 +138,6 @@ stringData:
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.secretKey: thisISaSECRET_1234
   connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
   # Only needed when using celery workers (instead of Kubernetes executors)
   connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow

examples/simple-airflow-cluster-ldap.yaml

@@ -136,7 +136,6 @@ stringData:
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.secretKey: thisISaSECRET_1234
   connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
   # Only needed when using celery workers (instead of Kubernetes executors)
   connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow

examples/simple-airflow-cluster.yaml

@@ -10,7 +10,6 @@ stringData:
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: airflow
-  connections.secretKey: thisISaSECRET_1234
   connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow
   # Only needed when using celery workers (instead of Kubernetes executors)
   connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql.default.svc.cluster.local/airflow

rust/operator-binary/Cargo.toml

@@ -13,18 +13,20 @@ product-config.workspace = true
 stackable-operator.workspace = true
 
 anyhow.workspace = true
+base64.workspace = true
 clap.workspace = true
 const_format.workspace = true
 fnv.workspace = true
 futures.workspace = true
+indoc.workspace = true
+rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_yaml.workspace = true
 snafu.workspace = true
 strum.workspace = true
 tokio.workspace = true
 tracing.workspace = true
-indoc.workspace = true
 
 [build-dependencies]
 built.workspace = true

rust/operator-binary/src/airflow_controller.rs

@@ -92,7 +92,9 @@ use crate::{
             AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved,
         },
         authorization::AirflowAuthorizationResolved,
-        build_recommended_labels, v1alpha1,
+        build_recommended_labels,
+        internal_secret::{ENV_INTERNAL_SECRET, ENV_JWT_SECRET, create_random_secret},
+        v1alpha1,
     },
     env_vars::{self, build_airflow_template_envs},
     operations::{
@@ -346,6 +348,9 @@ pub enum Error {
     ResolveProductImage {
         source: product_image_selection::Error,
     },
+
+    #[snafu(display("failed to create internal secret"))]
+    InvalidInternalSecret { source: crd::internal_secret::Error },
 }
 
 type Result<T, E = Error> = std::result::Result<T, E>;
@@ -470,6 +475,26 @@ pub async fn reconcile_airflow(
         .await?;
     }
 
+    create_random_secret(
+        airflow.shared_internal_secret_name().as_ref(),
+        ENV_INTERNAL_SECRET,
+        256,
+        airflow,
+        client,
+    )
+    .await
+    .context(InvalidInternalSecretSnafu)?;
+
+    create_random_secret(
+        airflow.shared_jwt_secret_name().as_ref(),
+        ENV_JWT_SECRET,
+        256,
+        airflow,
+        client,
+    )
+    .await
+    .context(InvalidInternalSecretSnafu)?;
+
     for (role_name, role_config) in validated_role_config.iter() {
         let airflow_role =
             AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu {

rust/operator-binary/src/crd/internal_secret.rs

@@ -0,0 +1,116 @@
+use std::collections::BTreeMap;
+
+use base64::{Engine as _, engine::general_purpose};
+use rand::{TryRngCore, rand_core::OsError, rngs::OsRng};
+use snafu::{OptionExt, ResultExt, Snafu};
+use stackable_operator::{
+    builder::meta::ObjectMetaBuilder, client::Client, k8s_openapi::api::core::v1::Secret,
+    kube::ResourceExt, logging::controller::ReconcilerError,
+};
+use strum::{EnumDiscriminants, IntoStaticStr};
+
+use crate::{airflow_controller::AIRFLOW_CONTROLLER_NAME, crd::v1alpha1};
+
+// Used for env-vars: AIRFLOW__WEBSERVER__SECRET_KEY, AIRFLOW__API__SECRET_KEY
+// N.B. AIRFLOW__WEBSERVER__SECRET_KEY is deprecated as of 3.0.2.
+// Secret key used to run the api server. It should be as random as possible.
+// It should be consistent across instances of the webserver. The webserver key
+// is also used to authorize requests to Celery workers when logs are retrieved.
+pub const ENV_INTERNAL_SECRET: &str = "INTERNAL_SECRET";
+// Used for env-var: AIRFLOW__API_AUTH__JWT_SECRET
+// Secret key used to encode and decode JWTs to authenticate to public and
+// private APIs. It should be as random as possible, but consistent across
+// instances of API services.
+pub const ENV_JWT_SECRET: &str = "JWT_SECRET";
+
+type Result<T, E = Error> = std::result::Result<T, E>;
+
+impl ReconcilerError for Error {
+    fn category(&self) -> &'static str {
+        ErrorDiscriminants::from(self).into()
+    }
+}
+
+#[derive(Snafu, Debug, EnumDiscriminants)]
+#[strum_discriminants(derive(IntoStaticStr))]
+pub enum Error {
+    #[snafu(display("object defines no namespace"))]
+    ObjectHasNoNamespace,
+
+    #[snafu(display("object is missing metadata to build owner reference"))]
+    ObjectMissingMetadataForOwnerRef {
+        source: stackable_operator::builder::meta::Error,
+    },
+
+    #[snafu(display("failed to retrieve secret for internal communications"))]
+    FailedToRetrieveInternalSecret {
+        source: stackable_operator::client::Error,
+    },
+
+    #[snafu(display("failed to apply internal secret"))]
+    ApplyInternalSecret {
+        source: stackable_operator::client::Error,
+    },
+
+    #[snafu(display("object defines no namespace"))]
+    SeedRandomGenerator { source: OsError },
+}
+
+pub async fn create_random_secret(
+    secret_name: &str,
+    secret_key: &str,
+    secret_byte_size: usize,
+    airflow: &v1alpha1::AirflowCluster,
+    client: &Client,
+) -> Result<()> {
+    let mut internal_secret = BTreeMap::new();
+    internal_secret.insert(secret_key.to_string(), get_random_base64(secret_byte_size)?);
+
+    let secret = Secret {
+        immutable: Some(true),
+        metadata: ObjectMetaBuilder::new()
+            .name(secret_name)
+            .namespace_opt(airflow.namespace())
+            .ownerreference_from_resource(airflow, None, Some(true))
+            .context(ObjectMissingMetadataForOwnerRefSnafu)?
+            .build(),
+        string_data: Some(internal_secret),
+        ..Secret::default()
+    };
+
+    if client
+        .get_opt::<Secret>(
+            &secret.name_any(),
+            secret
+                .namespace()
+                .as_deref()
+                .context(ObjectHasNoNamespaceSnafu)?,
+        )
+        .await
+        .context(FailedToRetrieveInternalSecretSnafu)?
+        .is_none()
+    {
+        client
+            .apply_patch(AIRFLOW_CONTROLLER_NAME, &secret, &secret)
+            .await
+            .context(ApplyInternalSecretSnafu)?;
+    }
+
+    Ok(())
+}
+
+fn get_random_base64(byte_size: usize) -> Result<String, Error> {
+    let mut buf = vec![0u8; byte_size];
+    // OsRng is a cryptographically secure pseudo-random number generator
+    // (CSPRNG) and also has no possible state to leak and cannot be
+    // improperly seeded. See: https://rust-random.github.io/book/guide-gen.html#cryptographically-secure-pseudo-random-number-generator
+    // and https://github.com/rust-random/rand/blob/master/SECURITY.md#specific-generators
+    // This call explicity returns a Result. An alternative would be to
+    // use let mut rng = StdRng::from_os_rng() and then use fill_bytes
+    // but this may *still* panic if the underlying (OS) mechanism fails
+    // for some reason, so keep the potential panic transparent.
+    OsRng
+        .try_fill_bytes(&mut buf)
+        .context(SeedRandomGeneratorSnafu)?;
+    Ok(general_purpose::STANDARD.encode(buf))
+}

rust/operator-binary/src/crd/mod.rs

@@ -60,6 +60,7 @@ use crate::{
 pub mod affinity;
 pub mod authentication;
 pub mod authorization;
+pub mod internal_secret;
 
 pub const APP_NAME: &str = "airflow";
 pub const OPERATOR_NAME: &str = "airflow.stackable.tech";
@@ -452,6 +453,14 @@ impl v1alpha1::AirflowCluster {
         tracing::debug!("Merged executor config: {:?}", conf_executor);
         fragment::validate(conf_executor).context(FragmentValidationFailureSnafu)
     }
+
+    pub fn shared_internal_secret_name(&self) -> String {
+        format!("{}-internal-secret", &self.name_any())
+    }
+
+    pub fn shared_jwt_secret_name(&self) -> String {
+        format!("{}-jwt-secret", &self.name_any())
+    }
 }
 
 fn extract_role_from_webserver_config(