refactor!: remove openlineage.enabled field

Author: razvan

docs/modules/spark-k8s/pages/usage-guide/openlineage.adoc

@@ -3,7 +3,7 @@
 https://openlineage.io/[OpenLineage] is an open standard for data lineage collection.
 The Spark operator can automatically emit lineage events for a `SparkApplication` to an OpenLineage-compatible backend such as https://marquezproject.github.io/marquez/[Marquez], without any manual `sparkConf` wiring.
 
-When enabled, the operator injects everything required to make the https://openlineage.io/docs/integrations/spark/[OpenLineage Spark listener] work:
+When configured, the operator injects everything required to make the https://openlineage.io/docs/integrations/spark/[OpenLineage Spark listener] work:
 
 * the OpenLineage Spark listener (appended to `spark.extraListeners`, so any listener you configure yourself is preserved),
 * the OpenLineage jar baked into the Spark image, referenced via `spark.jars` so it shares the same classloader as connectors pulled in through xref:usage-guide/job-dependencies.adoc[`deps.packages`] (for example Apache Iceberg),
@@ -23,14 +23,13 @@ kind: SparkApplication
 metadata:
   name: orders-by-nation
 spec:
-  openLineage:
-    enabled: true # <1>
+  openLineage: # <1>
     configMapName: marquez-discovery # <2>
     # namespace: orders-lineage # <3>
     # appName: orders-by-nation # <4>
   ...
 ----
-<1> Enable OpenLineage event emission. Defaults to `false`, in which case the operator emits nothing OpenLineage-related and behaviour is unchanged.
+<1> Adding the `openLineage` block enables OpenLineage event emission. Omit it entirely to leave OpenLineage off (the default), in which case the operator emits nothing OpenLineage-related and behaviour is unchanged.
 <2> Name of a discovery ConfigMap holding the backend endpoint (see below).
     Must be in the same namespace as the `SparkApplication`.
 <3> Optional. The OpenLineage namespace events are reported under.
@@ -52,7 +51,7 @@ data:
 <1> Base URL of the OpenLineage backend, for example the Marquez API service.
 
 Alternatively, you can set the endpoint directly through `sparkConf` (`spark.openlineage.transport.url`).
-If `enabled: true` and no endpoint can be resolved from either the discovery ConfigMap or `sparkConf`, reconciliation fails with a clear error rather than emitting a config that silently drops events.
+If the `openLineage` block is present and no endpoint can be resolved from either the discovery ConfigMap or `sparkConf`, reconciliation fails with a clear error rather than emitting a config that silently drops events.
 
 [#job-name]
 == Job name

extra/crds.yaml

@@ -1982,10 +1982,6 @@ spec:
                     nullable: true
                     pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                     type: string
-                  enabled:
-                    default: false
-                    description: Enable OpenLineage event emission. Defaults to `false` (nothing is injected).
-                    type: boolean
                   namespace:
                     description: |-
                       The OpenLineage namespace lineage is reported under.
@@ -6624,10 +6620,6 @@ spec:
                     nullable: true
                     pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                     type: string
-                  enabled:
-                    default: false
-                    description: Enable OpenLineage event emission. Defaults to `false` (nothing is injected).
-                    type: boolean
                   namespace:
                     description: |-
                       The OpenLineage namespace lineage is reported under.

rust/operator-binary/src/config/jvm.rs

@@ -39,12 +39,7 @@ pub fn construct_extra_java_options(
     // OpenLineage on Spark 4.x needs `java.base/java.security` opened to the unnamed module,
     // otherwise the driver throws a non-fatal `InaccessibleObjectException` and silently degrades
     // extension-interface lineage (MVP §7). Added to both driver and executor.
-    if spark_application
-        .spec
-        .open_lineage
-        .as_ref()
-        .is_some_and(|open_lineage| open_lineage.enabled)
-    {
+    if spark_application.spec.open_lineage.is_some() {
         jvm_args.push(OPENLINEAGE_ADD_OPENS.to_string());
     }
 
@@ -157,8 +152,7 @@ mod tests {
               mainApplicationFile: test.py
               sparkImage:
                 productVersion: 1.2.3
-              openLineage:
-                enabled: true
+              openLineage: {}
         "#;
 
         let deserializer = serde_yaml::Deserializer::from_str(input);

rust/operator-binary/src/crd/mod.rs

@@ -753,9 +753,7 @@ impl v1alpha1::SparkApplication {
         // `sparkConf` (so it goes in BEFORE the merge below). The two append-merge keys
         // (`spark.jars`, `spark.extraListeners`) are handled AFTER the merge so a user value is
         // combined with — not clobbered by — ours. See the OpenLineage usage guide.
-        if let Some(open_lineage) = &self.spec.open_lineage
-            && open_lineage.enabled
-        {
+        if let Some(open_lineage) = &self.spec.open_lineage {
             // Fail fast rather than emit a transport that silently drops every event.
             let has_url_override = self
                 .spec
@@ -795,12 +793,7 @@ impl v1alpha1::SparkApplication {
 
         // OpenLineage append-merge keys: combine our values with any user-provided ones (already
         // merged into `submit_conf` above) so neither clobbers the other.
-        if self
-            .spec
-            .open_lineage
-            .as_ref()
-            .is_some_and(|open_lineage| open_lineage.enabled)
-        {
+        if self.spec.open_lineage.is_some() {
             append_conf_csv(
                 &mut submit_conf,
                 "spark.extraListeners",
@@ -1670,8 +1663,7 @@ spec:
           mainApplicationFile: test.py
           sparkImage:
             productVersion: 1.2.3
-          openLineage:
-            enabled: true
+          openLineage: {}
     "#};
 
     #[test]
@@ -1701,36 +1693,22 @@ spec:
         );
     }
 
-    #[rstest]
-    #[case::absent(indoc! {r#"
-        ---
-        apiVersion: spark.stackable.tech/v1alpha1
-        kind: SparkApplication
-        metadata:
-          name: spark-examples
-          namespace: default
-        spec:
-          mode: cluster
-          mainApplicationFile: test.py
-          sparkImage:
-            productVersion: 1.2.3
-    "#})]
-    #[case::disabled(indoc! {r#"
-        ---
-        apiVersion: spark.stackable.tech/v1alpha1
-        kind: SparkApplication
-        metadata:
-          name: spark-examples
-          namespace: default
-        spec:
-          mode: cluster
-          mainApplicationFile: test.py
-          sparkImage:
-            productVersion: 1.2.3
-          openLineage:
-            enabled: false
-    "#})]
-    fn test_openlineage_injects_nothing_when_absent_or_disabled(#[case] yaml: &str) {
+    #[test]
+    fn test_openlineage_injects_nothing_when_absent() {
+        // No `openLineage` block → OpenLineage is off (its absence is the disable switch).
+        let yaml = indoc! {r#"
+            ---
+            apiVersion: spark.stackable.tech/v1alpha1
+            kind: SparkApplication
+            metadata:
+              name: spark-examples
+              namespace: default
+            spec:
+              mode: cluster
+              mainApplicationFile: test.py
+              sparkImage:
+                productVersion: 1.2.3
+        "#};
         // Endpoint is supplied to prove it is ignored, not that it is simply missing.
         let command = build_command_with_openlineage(yaml, Some("http://marquez:5000"));
 
@@ -1754,8 +1732,7 @@ spec:
               mainApplicationFile: test.py
               sparkImage:
                 productVersion: 1.2.3
-              openLineage:
-                enabled: true
+              openLineage: {}
               sparkConf:
                 spark.extraListeners: com.example.CustomListener
         "#};
@@ -1780,8 +1757,7 @@ spec:
               mainApplicationFile: test.py
               sparkImage:
                 productVersion: 1.2.3
-              openLineage:
-                enabled: true
+              openLineage: {}
               sparkConf:
                 spark.openlineage.transport.url: http://custom:1234
                 spark.openlineage.namespace: custom-ns
@@ -1818,8 +1794,7 @@ spec:
               mainApplicationFile: test.py
               sparkImage:
                 productVersion: 1.2.3
-              openLineage:
-                enabled: true
+              openLineage: {}
               sparkConf:
                 spark.openlineage.transport.url: http://custom:1234
         "#};
@@ -1843,7 +1818,6 @@ spec:
               sparkImage:
                 productVersion: 1.2.3
               openLineage:
-                enabled: true
                 appName: explicit-name
               sparkConf:
                 spark.app.name: spark-conf-name
@@ -1864,8 +1838,7 @@ spec:
               mainApplicationFile: test.py
               sparkImage:
                 productVersion: 1.2.3
-              openLineage:
-                enabled: true
+              openLineage: {}
               sparkConf:
                 spark.app.name: spark-conf-name
         "#},
@@ -1885,8 +1858,7 @@ spec:
               mainApplicationFile: test.py
               sparkImage:
                 productVersion: 1.2.3
-              openLineage:
-                enabled: true
+              openLineage: {}
         "#},
         "metadata-name",
         true

rust/operator-binary/src/openlineage.rs

@@ -15,18 +15,15 @@ use crate::crd::{Error, ObjectHasNoNameSnafu, v1alpha1};
 
 /// OpenLineage lineage emission for a [`SparkApplication`].
 ///
-/// When enabled, the operator injects the OpenLineage Spark listener, points its HTTP transport
-/// at the backend resolved from the discovery ConfigMap, and sets a stable job name. All injected
-/// values are defaults: they can be overridden via `sparkConf`.
+/// Adding this block enables OpenLineage: the operator injects the OpenLineage Spark listener,
+/// points its HTTP transport at the backend resolved from the discovery ConfigMap, and sets a
+/// stable job name. Omit the block to leave OpenLineage off. All injected values are defaults:
+/// they can be overridden via `sparkConf`.
 ///
 /// [`SparkApplication`]: crate::crd::v1alpha1::SparkApplication
 #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct OpenLineageSpec {
-    /// Enable OpenLineage event emission. Defaults to `false` (nothing is injected).
-    #[serde(default)]
-    pub enabled: bool,
-
     /// The OpenLineage namespace lineage is reported under.
     /// Defaults to the application's Kubernetes namespace (`metadata.namespace`).
     #[serde(default, skip_serializing_if = "Option::is_none")]

rust/operator-binary/src/spark_k8s_controller.rs

@@ -243,11 +243,7 @@ pub async fn reconcile(
     // warn if the job name falls back to `metadata.name` — both only matter when OpenLineage is
     // enabled.
     let openlineage_endpoint = resolve_openlineage_endpoint(client, spark_application).await?;
-    if spark_application
-        .spec
-        .open_lineage
-        .as_ref()
-        .is_some_and(|open_lineage| open_lineage.enabled)
+    if spark_application.spec.open_lineage.is_some()
         && spark_application
             .resolved_openlineage_app_name()
             .context(BuildCommandSnafu)?
@@ -341,8 +337,7 @@ async fn resolve_openlineage_endpoint(
     let Some(open_lineage) = &spark_application.spec.open_lineage else {
         return Ok(None);
     };
-    let (true, Some(config_map_name)) = (open_lineage.enabled, &open_lineage.config_map_name)
-    else {
+    let Some(config_map_name) = &open_lineage.config_map_name else {
         return Ok(None);
     };