refactor: remove pod_driver_controller.rs

Author: razvan

CHANGELOG.md

@@ -18,6 +18,7 @@ All notable changes to this project will be documented in this file.
 
 - BREAKING: The operator no longer runs a separate `spark-submit` process for `SparkApplication`s. The driver is launched directly as a Kubernetes `Job` (built from `spec.driver`) running `spark-submit` in client mode; executors are still created by the driver via Spark's Kubernetes backend. A headless driver `Service` is created so executors can reach the driver. This affects both `v1alpha1` (after conversion) and `v1alpha2` objects ([#711]).
 - The driver `Job` is no longer retried on failure (`backoffLimit` is `0`); the previous `spec.job.retryOnFailureCount` is deprecated and ignored ([#711]).
+- The `SparkApplication` status (`status.phase`) is now derived from the driver `Job` status by the application controller instead of a dedicated pod-driver controller watching driver pods. The pod-driver controller and the operator's `pods` RBAC permissions have been removed ([#711]).
 - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#674]).
 - BREAKING: Each custom resource accepts now only the known config files in `configOverrides`:
   - `SparkApplication`: `spark-env.sh` and `security.properties`

deploy/helm/spark-k8s-operator/templates/roles.yaml

@@ -13,17 +13,6 @@ rules:
       - nodes/proxy
     verbs:
       - get
-  # The pod-driver controller watches Spark driver pods (labelled spark-role=driver) to track
-  # SparkApplication completion. Driver pods are owned by the driver Job and cleaned up via the
-  # Job's ttlSecondsAfterFinished, so the operator no longer deletes them.
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-      - list
-      - watch
   # ConfigMaps hold pod templates and Spark configuration. All three controllers apply
   # them via Server-Side Apply (create + patch). The history and connect controllers
   # track them for orphan cleanup (list + delete). All controllers watch ConfigMaps via

rust/operator-binary/src/crd/constants.rs

@@ -66,10 +66,6 @@ pub const FIELD_MANAGER: &str = "spark-operator";
 pub const SPARK_CONTROLLER_NAME: &str = "sparkapplication";
 pub const SPARK_FULL_CONTROLLER_NAME: &str = concatcp!(SPARK_CONTROLLER_NAME, '.', OPERATOR_NAME);
 
-pub const POD_DRIVER_CONTROLLER_NAME: &str = "pod-driver";
-pub const POD_DRIVER_FULL_CONTROLLER_NAME: &str =
-    concatcp!(POD_DRIVER_CONTROLLER_NAME, '.', OPERATOR_NAME);
-
 pub const HISTORY_CONTROLLER_NAME: &str = "history";
 pub const HISTORY_FULL_CONTROLLER_NAME: &str =
     concatcp!(HISTORY_CONTROLLER_NAME, '.', OPERATOR_NAME);

rust/operator-binary/src/main.rs

@@ -16,7 +16,7 @@ use stackable_operator::{
     k8s_openapi::api::{
         apps::v1::StatefulSet,
         batch::v1::Job,
-        core::v1::{ConfigMap, Pod, Service},
+        core::v1::{ConfigMap, Service},
     },
     kube::{
         CustomResourceExt as _,
@@ -40,8 +40,7 @@ use crate::{
     crd::{
         SparkApplication,
         constants::{
-            HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, POD_DRIVER_FULL_CONTROLLER_NAME,
-            SPARK_CONTROLLER_NAME, SPARK_FULL_CONTROLLER_NAME,
+            HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, SPARK_FULL_CONTROLLER_NAME,
         },
         history::SparkHistoryServer,
         template_spec::{SparkApplicationTemplate, SparkApplicationTemplateVersion},
@@ -53,7 +52,6 @@ mod config;
 mod connect;
 mod crd;
 mod history;
-mod pod_driver_controller;
 mod product_logging;
 mod spark_k8s_controller;
 mod webhooks;
@@ -199,47 +197,6 @@ async fn main() -> anyhow::Result<()> {
             )
             .map(anyhow::Ok);
 
-            let pod_driver_event_recorder = Arc::new(Recorder::new(
-                client.as_kube_client(),
-                Reporter {
-                    controller: POD_DRIVER_FULL_CONTROLLER_NAME.to_string(),
-                    instance: None,
-                },
-            ));
-            let pod_driver_controller = Controller::new(
-                watch_namespace.get_api::<DeserializeGuard<Pod>>(&client),
-                watcher::Config::default()
-                 .labels(&format!("app.kubernetes.io/managed-by={OPERATOR_NAME}_{SPARK_CONTROLLER_NAME},spark-role=driver")),
-            )
-            .owns(
-                watch_namespace.get_api::<DeserializeGuard<Pod>>(&client),
-                watcher::Config::default(),
-            )
-            .graceful_shutdown_on(sigterm_watcher.handle())
-            .run(
-                pod_driver_controller::reconcile,
-                pod_driver_controller::error_policy,
-                Arc::new(client.clone()),
-            )
-            .instrument(info_span!("pod_driver_controller"))
-            // We can let the reporting happen in the background
-            .for_each_concurrent(
-                16, // concurrency limit
-                |result| {
-                    // The event_recorder needs to be shared across all invocations, so that
-                    // events are correctly aggregated
-                    let pod_driver_event_recorder = pod_driver_event_recorder.clone();
-                    async move {
-                        report_controller_reconciled(
-                            &pod_driver_event_recorder,
-                            POD_DRIVER_FULL_CONTROLLER_NAME,
-                            &result,
-                        )
-                        .await;
-                    }
-                },
-            ).map(anyhow::Ok);
-
             // Create new object because Ctx cannot be cloned
             let ctx = Ctx {
                 client: client.clone(),
@@ -399,7 +356,6 @@ async fn main() -> anyhow::Result<()> {
 
             // kube-runtime's Controller will tokio::spawn each reconciliation, so this only concerns the internal watch machinery
             futures::try_join!(
-                pod_driver_controller,
                 delayed_history_controller,
                 delayed_connect_controller,
                 delayed_app_controller,