wip: airflow versioning

Author: adwk67

Cargo.lock

@@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator"
 
 [workspace.dependencies]
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.3" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned", "webhook"], tag = "stackable-operator-0.100.3" }
 
 anyhow = "1.0"
 base64 = "0.22"

Cargo.nix

@@ -4,6 +4,9 @@ kind: Deployment
 metadata:
   name: {{ include "operator.fullname" . }}-deployment
   labels:
+    {{- if .Values.maintenance.customResourceDefinitions.maintain }}
+    webhook.stackable.tech/conversion: enabled
+    {{- end }}
     {{- include "operator.labels" . | nindent 4 }}
 spec:
   replicas: 1
@@ -79,6 +82,7 @@ spec:
               value: {{ .Values.kubernetesClusterDomain | quote }}
             {{- end }}
             {{- include "telemetry.envVars" . | nindent 12 }}
+            {{- include "maintenance.envVars" . | nindent 12 }}
       volumes:
         - name: config-spec
           configMap:

Cargo.toml

@@ -91,6 +91,12 @@ rules:
       - customresourcedefinitions
     verbs:
       - get
+      # Required to maintain the CRD. The operator needs to do this, as it needs to enter e.g. it's
+      # generated certificate in the conversion webhook.
+      {{- if .Values.maintenance.customResourceDefinitions.maintain }}
+      - create
+      - patch
+      {{- end }}
   - apiGroups:
       - listeners.stackable.tech
     resources:
@@ -111,6 +117,9 @@ rules:
       - list
       - patch
       - watch
+      {{- if .Values.maintenance.customResourceDefinitions.maintain }}
+      - create
+      {{- end }}
   - apiGroups:
       - {{ include "operator.name" . }}.stackable.tech
     resources:

crate-hashes.json

@@ -1,4 +1,4 @@
-
+{{- if .Values.maintenance.customResourceDefinitions.maintain }}
 ---
 apiVersion: v1
 kind: Service
@@ -11,9 +11,11 @@ metadata:
     {{- include "operator.labels" . | nindent 4 }}
 spec:
   selector:
+    webhook.stackable.tech/conversion: enabled
     {{- include "operator.selectorLabels" . | nindent 6 }}
   ports:
     - name: conversion-webhook
       protocol: TCP
       port: 8443
       targetPort: 8443
+{{- end }}

deploy/helm/airflow-operator/crds/crds.yaml

@@ -46,6 +46,15 @@ affinity: {}
 # See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details.
 # kubernetesClusterDomain: my-cluster.local
 
+maintenance:
+  endOfSupportCheck:
+    enabled: true
+    # mode: offline
+    # interval: 24h
+
+  customResourceDefinitions:
+    maintain: true
+
 # See all available options and detailed explanations about the concept here:
 # https://docs.stackable.tech/home/stable/concepts/telemetry/
 telemetry:

deploy/helm/airflow-operator/templates/deployment.yaml

@@ -1,4 +1,4 @@
-//! Ensures that `Pod`s are configured and running for each [`v1alpha1::AirflowCluster`]
+//! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`]
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap},
     io::Write,
@@ -97,7 +97,7 @@ use crate::{
             FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY,
             create_random_secret,
         },
-        v1alpha1,
+        v1alpha2,
     },
     env_vars::{self, build_airflow_template_envs},
     operations::{
@@ -137,19 +137,19 @@ pub enum Error {
     #[snafu(display("failed to apply Service for {rolegroup}"))]
     ApplyRoleGroupService {
         source: stackable_operator::cluster_resources::Error,
-        rolegroup: RoleGroupRef<v1alpha1::AirflowCluster>,
+        rolegroup: RoleGroupRef<v1alpha2::AirflowCluster>,
     },
 
     #[snafu(display("failed to apply ConfigMap for {rolegroup}"))]
     ApplyRoleGroupConfig {
         source: stackable_operator::cluster_resources::Error,
-        rolegroup: RoleGroupRef<v1alpha1::AirflowCluster>,
+        rolegroup: RoleGroupRef<v1alpha2::AirflowCluster>,
     },
 
     #[snafu(display("failed to apply StatefulSet for {rolegroup}"))]
     ApplyRoleGroupStatefulSet {
         source: stackable_operator::cluster_resources::Error,
-        rolegroup: RoleGroupRef<v1alpha1::AirflowCluster>,
+        rolegroup: RoleGroupRef<v1alpha2::AirflowCluster>,
     },
 
     #[snafu(display("invalid product config"))]
@@ -200,13 +200,13 @@ pub enum Error {
     #[snafu(display("failed to build config file for {rolegroup}"))]
     BuildRoleGroupConfigFile {
         source: FlaskAppConfigWriterError,
-        rolegroup: RoleGroupRef<v1alpha1::AirflowCluster>,
+        rolegroup: RoleGroupRef<v1alpha2::AirflowCluster>,
     },
 
     #[snafu(display("failed to build ConfigMap for {rolegroup}"))]
     BuildRoleGroupConfig {
         source: stackable_operator::builder::configmap::Error,
-        rolegroup: RoleGroupRef<v1alpha1::AirflowCluster>,
+        rolegroup: RoleGroupRef<v1alpha2::AirflowCluster>,
     },
 
     #[snafu(display("failed to resolve and merge config for role and role group"))]
@@ -365,7 +365,7 @@ impl ReconcilerError for Error {
 }
 
 pub async fn reconcile_airflow(
-    airflow: Arc<DeserializeGuard<v1alpha1::AirflowCluster>>,
+    airflow: Arc<DeserializeGuard<v1alpha2::AirflowCluster>>,
     ctx: Arc<Ctx>,
 ) -> Result<Action> {
     tracing::info!("Starting reconcile");
@@ -682,7 +682,7 @@ pub async fn reconcile_airflow(
 
 #[allow(clippy::too_many_arguments)]
 async fn build_executor_template(
-    airflow: &v1alpha1::AirflowCluster,
+    airflow: &v1alpha2::AirflowCluster,
     common_config: &CommonConfiguration<ExecutorConfigFragment, GenericProductSpecificCommonConfig>,
     resolved_product_image: &ResolvedProductImage,
     authentication_config: &AirflowClientAuthenticationDetailsResolved,
@@ -750,9 +750,9 @@ async fn build_executor_template(
 /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator
 #[allow(clippy::too_many_arguments)]
 fn build_rolegroup_config_map(
-    airflow: &v1alpha1::AirflowCluster,
+    airflow: &v1alpha2::AirflowCluster,
     resolved_product_image: &ResolvedProductImage,
-    rolegroup: &RoleGroupRef<v1alpha1::AirflowCluster>,
+    rolegroup: &RoleGroupRef<v1alpha2::AirflowCluster>,
     rolegroup_config: &HashMap<PropertyNameKind, BTreeMap<String, String>>,
     authentication_config: &AirflowClientAuthenticationDetailsResolved,
     authorization_config: &AirflowAuthorizationResolved,
@@ -863,9 +863,9 @@ fn build_rolegroup_config_map(
 }
 
 fn build_rolegroup_metadata(
-    airflow: &v1alpha1::AirflowCluster,
+    airflow: &v1alpha2::AirflowCluster,
     resolved_product_image: &&ResolvedProductImage,
-    rolegroup: &&RoleGroupRef<v1alpha1::AirflowCluster>,
+    rolegroup: &&RoleGroupRef<v1alpha2::AirflowCluster>,
     prometheus_label: Label,
     name: String,
 ) -> Result<ObjectMeta, Error> {
@@ -888,8 +888,8 @@ fn build_rolegroup_metadata(
 }
 
 pub fn build_group_listener(
-    airflow: &v1alpha1::AirflowCluster,
-    object_labels: ObjectLabels<v1alpha1::AirflowCluster>,
+    airflow: &v1alpha2::AirflowCluster,
+    object_labels: ObjectLabels<v1alpha2::AirflowCluster>,
     listener_class: String,
     listener_group_name: String,
 ) -> Result<listener::v1alpha1::Listener> {
@@ -924,10 +924,10 @@ fn listener_ports() -> Vec<listener::v1alpha1::ListenerPort> {
 /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator.
 #[allow(clippy::too_many_arguments)]
 fn build_server_rolegroup_statefulset(
-    airflow: &v1alpha1::AirflowCluster,
+    airflow: &v1alpha2::AirflowCluster,
     resolved_product_image: &ResolvedProductImage,
     airflow_role: &AirflowRole,
-    rolegroup_ref: &RoleGroupRef<v1alpha1::AirflowCluster>,
+    rolegroup_ref: &RoleGroupRef<v1alpha2::AirflowCluster>,
     rolegroup_config: &HashMap<PropertyNameKind, BTreeMap<String, String>>,
     authentication_config: &AirflowClientAuthenticationDetailsResolved,
     authorization_config: &AirflowAuthorizationResolved,
@@ -1247,14 +1247,14 @@ fn build_logging_container(
 
 #[allow(clippy::too_many_arguments)]
 fn build_executor_template_config_map(
-    airflow: &v1alpha1::AirflowCluster,
+    airflow: &v1alpha2::AirflowCluster,
     resolved_product_image: &ResolvedProductImage,
     authentication_config: &AirflowClientAuthenticationDetailsResolved,
     sa_name: &str,
     merged_executor_config: &ExecutorConfig,
     env_overrides: &HashMap<String, String>,
     pod_overrides: &PodTemplateSpec,
-    rolegroup_ref: &RoleGroupRef<v1alpha1::AirflowCluster>,
+    rolegroup_ref: &RoleGroupRef<v1alpha2::AirflowCluster>,
     git_sync_resources: &git_sync::v1alpha2::GitSyncResources,
 ) -> Result<ConfigMap> {
     let mut pb = PodBuilder::new();
@@ -1386,7 +1386,7 @@ fn build_executor_template_config_map(
 }
 
 pub fn error_policy(
-    _obj: Arc<DeserializeGuard<v1alpha1::AirflowCluster>>,
+    _obj: Arc<DeserializeGuard<v1alpha2::AirflowCluster>>,
     error: &Error,
     _ctx: Arc<Ctx>,
 ) -> Action {