Merge branch 'main' into chore/connect-kerberos-iceberg

razvan · web-flow · commit 636d92c7651f · 2026-06-25T15:43:08.000+02:00
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -48,7 +48,7 @@ jobs:
 
       - name: Check for changed files
         id: check
-        uses: stackabletech/actions/detect-changes@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+        uses: stackabletech/actions/detect-changes@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           patterns: |
             - '.github/workflows/build.yaml'
@@ -166,16 +166,16 @@ jobs:
 
       - name: Build Container Image
         id: build
-        uses: stackabletech/actions/build-container-image@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+        uses: stackabletech/actions/build-container-image@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           image-name: ${{ env.OPERATOR_NAME }}
           image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }}
           build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }}
           container-file: docker/Dockerfile
 
-      - name: Publish Container Image
+      - name: Publish Container Image to oci.stackable.tech
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: stackabletech/actions/publish-image@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+        uses: stackabletech/actions/publish-image@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$sdp+github-action-build
@@ -184,6 +184,17 @@ jobs:
           image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
           source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
 
+      - name: Publish Container Image to quay.io
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        uses: stackabletech/actions/publish-image@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
+        with:
+          image-registry-uri: quay.io
+          image-registry-username: stackable+robot_sdp_github_action_build
+          image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: stackable/sdp/${{ env.OPERATOR_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
   publish-index-manifest:
     name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index
     if: |
@@ -202,15 +213,24 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Publish and Sign Image Index
-        uses: stackabletech/actions/publish-image-index-manifest@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+      - name: Publish and Sign Image Index to oci.stackable.tech
+        uses: stackabletech/actions/publish-image-index-manifest@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           image-registry-uri: oci.stackable.tech
           image-registry-username: robot$sdp+github-action-build
           image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
           image-repository: sdp/${{ env.OPERATOR_NAME }}
           image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }}
 
+      - name: Publish and Sign Image Index to quay.io
+        uses: stackabletech/actions/publish-image-index-manifest@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
+        with:
+          image-registry-uri: quay.io
+          image-registry-username: stackable+robot_sdp_github_action_build
+          image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: stackable/sdp/${{ env.OPERATOR_NAME }}
+          image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }}
+
   publish-helm-chart:
     name: Package/Publish ${{ needs.build-container-image.outputs.operator-version }} Helm Chart
     if: |
@@ -229,8 +249,8 @@ jobs:
           persist-credentials: false
           submodules: recursive
 
-      - name: Package, Publish, and Sign Helm Chart
-        uses: stackabletech/actions/publish-helm-chart@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+      - name: Package, Publish, and Sign Helm Chart to oci.stackable.tech
+        uses: stackabletech/actions/publish-helm-chart@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           chart-registry-uri: oci.stackable.tech
           chart-registry-username: robot$sdp-charts+github-action-build
@@ -241,6 +261,19 @@ jobs:
           app-version: ${{ needs.build-container-image.outputs.operator-version }}
           publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }}
 
+      - name: Package, Publish, and Sign Helm Chart to quay.io
+        uses: stackabletech/actions/publish-helm-chart@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
+        with:
+          chart-registry-uri: quay.io
+          chart-registry-username: stackable+robot_sdp_charts_github_action_build
+          chart-registry-password: ${{ secrets.QUAY_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }}
+          chart-repository: stackable/sdp-charts
+          chart-directory: deploy/helm/${{ env.OPERATOR_NAME }}
+          chart-version: ${{ needs.build-container-image.outputs.operator-version }}
+          app-version: ${{ needs.build-container-image.outputs.operator-version }}
+          publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }}
+          helm-version: v3.17.4 # This is currently the latest version which supports pushing to quay.io
+
   openshift-preflight-check:
     name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }}
     if: |
@@ -259,12 +292,18 @@ jobs:
           - arm64
     runs-on: ubuntu-latest
     steps:
-      - name: Run OpenShift Preflight Check
-        uses: stackabletech/actions/run-openshift-preflight@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+      - name: Run OpenShift Preflight Check for oci.stackable.tech
+        uses: stackabletech/actions/run-openshift-preflight@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }}
           image-architecture: ${{ matrix.arch }}
 
+      - name: Run OpenShift Preflight Check for quay.io
+        uses: stackabletech/actions/run-openshift-preflight@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
+        with:
+          image-index-uri: quay.io/stackable/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }}
+          image-architecture: ${{ matrix.arch }}
+
   # This job is a required check in GitHub Settings for this repository.
   # It saves us having to list many required jobs, or work around dynamically
   # named jobs (since there is no concept of required settings).
@@ -300,7 +339,7 @@ jobs:
           persist-credentials: false
 
       - name: Send Notification
-        uses: stackabletech/actions/send-slack-notification@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+        uses: stackabletech/actions/send-slack-notification@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           publish-helm-chart-result: ${{ needs.publish-helm-chart.result }}
           publish-manifests-result: ${{ needs.publish-index-manifest.result }}
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -41,7 +41,7 @@ jobs:
       # TODO: Enable the scheduled runs which hard-code what profile to use
       - name: Run Integration Test
         id: test
-        uses: stackabletech/actions/run-integration-test@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+        uses: stackabletech/actions/run-integration-test@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           replicated-api-token: ${{ secrets.REPLICATED_API_TOKEN }}
           test-mode-input: ${{ inputs.test-mode-input }}
@@ -51,7 +51,7 @@ jobs:
 
       - name: Send Notification
         if: ${{ failure() || github.run_attempt > 1 }}
-        uses: stackabletech/actions/send-slack-notification@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+        uses: stackabletech/actions/send-slack-notification@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           slack-token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }}
           failed-tests: ${{ steps.test.outputs.failed-tests }}
diff --git a/.github/workflows/pr_prek.yaml b/.github/workflows/pr_prek.yaml
@@ -1,5 +1,5 @@
 ---
-name: pre-commit
+name: prek
 
 on:
   pull_request:
@@ -14,7 +14,7 @@ env:
   JINJA2_CLI_VERSION: "1.0.0"
 
 jobs:
-  pre-commit:
+  prek:
     runs-on: ubuntu-latest
     steps:
       - name: Install host dependencies
@@ -27,7 +27,7 @@ jobs:
           persist-credentials: false
           submodules: recursive
           fetch-depth: 0
-      - uses: stackabletech/actions/run-pre-commit@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
+      - uses: stackabletech/actions/run-prek@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           rust: ${{ env.RUST_TOOLCHAIN_VERSION }}
diff --git a/.readme/partials/borrowed/overview_blurb.md.j2 b/.readme/partials/borrowed/overview_blurb.md.j2
@@ -1,2 +1,2 @@
 <!-- markdownlint-disable MD041 MD051 -->
-It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
+It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka®, OpenSearch, Trino or Apache Spark™, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,14 +24,18 @@ All notable changes to this project will be documented in this file.
 - Bump `stackable-operator` to 0.110.1 ([#679]).
 - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler for spark application, spark connect and spark history server([#687]).
 - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#689]).
+- Fix the `SparkApplication` CRD description, which incorrectly described it as a "Spark cluster stacklet" rather than a Spark application ([#705]).
+- BREAKING: make application templates namespaced instead of cluster wide objects ([#694]).
 
 [#674]: https://github.com/stackabletech/spark-k8s-operator/pull/674
 [#679]: https://github.com/stackabletech/spark-k8s-operator/pull/679
 [#680]: https://github.com/stackabletech/spark-k8s-operator/pull/680
 [#684]: https://github.com/stackabletech/spark-k8s-operator/pull/684
 [#687]: https://github.com/stackabletech/spark-k8s-operator/pull/687
 [#689]: https://github.com/stackabletech/spark-k8s-operator/pull/689
+[#694]: https://github.com/stackabletech/spark-k8s-operator/pull/694
 [#696]: https://github.com/stackabletech/spark-k8s-operator/pull/696
+[#705]: https://github.com/stackabletech/spark-k8s-operator/pull/705
 
 ## [26.3.0] - 2026-03-16
 
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@
 This is a Kubernetes operator to manage [Apache Spark](https://spark.apache.org/) jobs.
 
 <!-- markdownlint-disable MD041 MD051 -->
-It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
+It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka®, OpenSearch, Trino or Apache Spark™, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
 
 ## Installation
 
diff --git a/deny.toml b/deny.toml
@@ -45,6 +45,25 @@ ignore = [
     # This version is pulled in transitively via num-bigint-dig -> rsa -> stackable-certs and cannot be
     # updated until the upstream rsa crate bumps its rand dependency.
     "RUSTSEC-2026-0097",
+
+    # https://rustsec.org/advisories/RUSTSEC-2026-0173
+    # The author of `proc-macro-error2` has [confirmed](https://github.com/GnomedDev/proc-macro-error-2/issues/17#issuecomment-4643215473)
+    # that the crate is no longer maintained and recommends that users migrate away from it.
+    #
+    # There currently is no way for us to negate this advisory, because that crate is not used
+    # directly by us. We need to wait for new versions of oci-spec and getset. See the following
+    # issue which tracks moving to a newer getset version: https://github.com/youki-dev/oci-spec-rs/issues/340
+    #
+    # proc-macro-error2 v2.0.1
+    #  └── getset v0.1.6
+    #      └── oci-spec v0.9.0
+    #          └── boil v0.2.1
+    #
+    # Alternate crates are:
+    #
+    # - https://crates.io/crates/manyhow
+    # - https://github.com/SergioBenitez/proc-macro2-diagnostics
+    "RUSTSEC-2026-0173",
 ]
 
 [bans]
diff --git a/deploy/helm/spark-k8s-operator/values/quay.io.yaml b/deploy/helm/spark-k8s-operator/values/quay.io.yaml
@@ -0,0 +1,4 @@
+---
+# Values overlay for chart packages published to quay.io.
+image:
+  repository: quay.io/stackable/sdp
diff --git a/docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc b/docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc
@@ -5,13 +5,23 @@ Spark application templates are used to define reusable configurations for Spark
 When you have many applications with similar configurations, templates can help you avoid duplication by grouping common settings together.
 Application templates are available for the `v1alpha1` version of the SparkApplication custom resource and share the exact same structure as the SparkApplication resource, but with some differences in the way the operator handles them:
 
-1. Application templates are cluster wide resources, while Spark application resources are namespace-scoped.
-  This means that application templates can be used across multiple namespaces, while Spark application resources are limited to the namespace they are created in.
+1. Application templates are namespace-scoped resources, just like Spark applications.
+  This means that a SparkApplication can only reference templates from its own namespace.
 2. Application templates are not reconciled by the operator, but must be referenced from a SparkApplication resource to be applied. This means that changes to an application template will not automatically trigger updates to SparkApplication resources that reference it.
 3. An application can reference multiple application templates, and the settings from these templates will be merged together. The merging order of the templates is indicated by their index in the reference list. The application fields have the highest precedence and will override any conflicting settings from the templates. This allows you to have a base template with common settings and then override specific settings in the application resource as needed.
 4. Application template references are immutable in the sense that once applied to an application they cannot be changed again. Currently templates are applied upon the creation of the application, and any changes to the template references after that will be ignored.
 5. Application and template CRDs must have the exact same versions. Currently only `v1alpha1` is supported.
 
+== Migrating from cluster-scoped templates
+
+IMPORTANT: Application templates used to be cluster wide resources when they were first released. This was a mistake. Many users do not have the access rights to create cluster scoped resources and so the templates are now namespace scoped.
+
+If you are migrating from older installations where templates were treated as cluster-wide resources, account for the following:
+
+1. Recreate each template in every namespace where SparkApplications use it.
+2. Keep template names consistent per namespace if you want the same application annotations to continue working.
+3. Cross-namespace template references are no longer resolved; templates and applications must be in the same namespace.
+4. Update GitOps/automation manifests to create templates as namespace-targeted resources before reconciling dependent SparkApplications.
 == Examples
 
 Applications use `metadata.annotations` to reference application templates as shown below:
diff --git a/extra/crds.yaml b/extra/crds.yaml
@@ -22,7 +22,7 @@ spec:
         properties:
           spec:
             description: |-
-              A Spark cluster stacklet. This resource is managed by the Stackable operator for Apache Spark.
+              A Spark application run on Kubernetes by the Stackable operator for Apache Spark.
               Find more information on how to use it and the resources that the operator generates in the
               [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/).
 
@@ -4639,7 +4639,7 @@ spec:
     shortNames:
     - sparkapptemplate
     singular: sparkapplicationtemplate
-  scope: Cluster
+  scope: Namespaced
   versions:
   - additionalPrinterColumns: []
     name: v1alpha1
diff --git a/renovate.json b/renovate.json
@@ -3,5 +3,5 @@
   "extends": [
     "local>stackabletech/.github:renovate-config"
   ],
-  "ignorePaths": [".github/workflows/build.yaml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_pre-commit.yaml"]
+  "ignorePaths": [".github/workflows/build.yaml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_prek.yaml"]
 }
diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs
@@ -169,7 +169,7 @@ pub mod versioned {
         pub resolved_template_ref: Vec<ResolvedSparkApplicationTemplate>,
     }
 
-    /// A Spark cluster stacklet. This resource is managed by the Stackable operator for Apache Spark.
+    /// A Spark application run on Kubernetes by the Stackable operator for Apache Spark.
     /// Find more information on how to use it and the resources that the operator generates in the
     /// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/).
     ///
diff --git a/rust/operator-binary/src/crd/template_merger.rs b/rust/operator-binary/src/crd/template_merger.rs
@@ -312,6 +312,47 @@ mod tests {
         assert_eq!(merged.spec.args, vec!["arg1", "arg2"]);
     }
 
+    #[test]
+    fn test_deep_merge_metadata_namespace_overlay_wins() {
+        let base = serde_yaml::from_str::<crate::crd::v1alpha1::SparkApplication>(indoc! {r#"
+            ---
+            apiVersion: spark.stackable.tech/v1alpha1
+            kind: SparkApplication
+            metadata:
+              name: base-app
+              namespace: template-namespace
+            spec:
+              mode: cluster
+              mainApplicationFile: base.py
+              sparkImage:
+                productVersion: "3.5.0"
+        "#})
+        .unwrap();
+
+        let overlay = serde_yaml::from_str::<crate::crd::v1alpha1::SparkApplication>(indoc! {r#"
+            ---
+            apiVersion: spark.stackable.tech/v1alpha1
+            kind: SparkApplication
+            metadata:
+              name: overlay-app
+              namespace: app-namespace
+            spec:
+              mode: cluster
+              mainApplicationFile: overlay.py
+              sparkImage:
+                productVersion: "3.5.1"
+        "#})
+        .unwrap();
+
+        let merged = deep_merge(&base, &overlay);
+
+        assert_eq!(
+            merged.metadata.namespace,
+            Some("app-namespace".to_string()),
+            "overlay namespace should take precedence"
+        );
+    }
+
     #[test]
     fn test_deep_merge_spark_conf() {
         let base = serde_yaml::from_str::<crate::crd::v1alpha1::SparkApplication>(indoc! {r#"
diff --git a/rust/operator-binary/src/crd/template_spec.rs b/rust/operator-binary/src/crd/template_spec.rs

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`<!-- markdownlint-disable MD041 MD051 -->`
`2`		`-It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).`
	`2`	`+It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka®, OpenSearch, Trino or Apache Spark™, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +---
 +# Values overlay for chart packages published to quay.io.
 +image:
 +  repository: quay.io/stackable/sdp