Skip to content

Commit e6ef4a7

Browse files
adwk67adwk67
committed
fix merge conflicts
2 parents e6001c0 + 5f6c983 commit e6ef4a7

51 files changed

Lines changed: 1474 additions & 45 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yaml‎

Lines changed: 54 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848

4949
- name: Check for changed files
5050
id: check
51-
uses: stackabletech/actions/detect-changes@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
51+
uses: stackabletech/actions/detect-changes@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
5252
with:
5353
patterns: |
5454
- '.github/workflows/build.yaml'
@@ -83,9 +83,8 @@ jobs:
8383
submodules: recursive
8484

8585
- name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} toolchain
86-
uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
87-
with:
88-
toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
86+
shell: bash
87+
run: rustup toolchain install "${RUST_TOOLCHAIN_VERSION}"
8988

9089
- name: Setup Rust Cache
9190
uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
@@ -160,22 +159,21 @@ jobs:
160159
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31.5.2
161160

162161
- name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} Toolchain
163-
uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
164-
with:
165-
toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
162+
shell: bash
163+
run: rustup toolchain install "${RUST_TOOLCHAIN_VERSION}"
166164

167165
- name: Build Container Image
168166
id: build
169-
uses: stackabletech/actions/build-container-image@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
167+
uses: stackabletech/actions/build-container-image@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
170168
with:
171169
image-name: ${{ env.OPERATOR_NAME }}
172170
image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }}
173171
build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }}
174172
container-file: docker/Dockerfile
175173

176-
- name: Publish Container Image
174+
- name: Publish Container Image to oci.stackable.tech
177175
if: ${{ !github.event.pull_request.head.repo.fork }}
178-
uses: stackabletech/actions/publish-image@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
176+
uses: stackabletech/actions/publish-image@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
179177
with:
180178
image-registry-uri: oci.stackable.tech
181179
image-registry-username: robot$sdp+github-action-build
@@ -184,6 +182,17 @@ jobs:
184182
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
185183
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
186184

185+
- name: Publish Container Image to quay.io
186+
if: ${{ !github.event.pull_request.head.repo.fork }}
187+
uses: stackabletech/actions/publish-image@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
188+
with:
189+
image-registry-uri: quay.io
190+
image-registry-username: stackable+robot_sdp_github_action_build
191+
image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
192+
image-repository: stackable/sdp/${{ env.OPERATOR_NAME }}
193+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
194+
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
195+
187196
publish-index-manifest:
188197
name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index
189198
if: |
@@ -202,15 +211,24 @@ jobs:
202211
with:
203212
persist-credentials: false
204213

205-
- name: Publish and Sign Image Index
206-
uses: stackabletech/actions/publish-image-index-manifest@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
214+
- name: Publish and Sign Image Index to oci.stackable.tech
215+
uses: stackabletech/actions/publish-image-index-manifest@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
207216
with:
208217
image-registry-uri: oci.stackable.tech
209218
image-registry-username: robot$sdp+github-action-build
210219
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
211220
image-repository: sdp/${{ env.OPERATOR_NAME }}
212221
image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }}
213222

223+
- name: Publish and Sign Image Index to quay.io
224+
uses: stackabletech/actions/publish-image-index-manifest@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
225+
with:
226+
image-registry-uri: quay.io
227+
image-registry-username: stackable+robot_sdp_github_action_build
228+
image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
229+
image-repository: stackable/sdp/${{ env.OPERATOR_NAME }}
230+
image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }}
231+
214232
publish-helm-chart:
215233
name: Package/Publish ${{ needs.build-container-image.outputs.operator-version }} Helm Chart
216234
if: |
@@ -229,8 +247,8 @@ jobs:
229247
persist-credentials: false
230248
submodules: recursive
231249

232-
- name: Package, Publish, and Sign Helm Chart
233-
uses: stackabletech/actions/publish-helm-chart@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
250+
- name: Package, Publish, and Sign Helm Chart to oci.stackable.tech
251+
uses: stackabletech/actions/publish-helm-chart@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
234252
with:
235253
chart-registry-uri: oci.stackable.tech
236254
chart-registry-username: robot$sdp-charts+github-action-build
@@ -241,6 +259,19 @@ jobs:
241259
app-version: ${{ needs.build-container-image.outputs.operator-version }}
242260
publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }}
243261

262+
- name: Package, Publish, and Sign Helm Chart to quay.io
263+
uses: stackabletech/actions/publish-helm-chart@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
264+
with:
265+
chart-registry-uri: quay.io
266+
chart-registry-username: stackable+robot_sdp_charts_github_action_build
267+
chart-registry-password: ${{ secrets.QUAY_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }}
268+
chart-repository: stackable/sdp-charts
269+
chart-directory: deploy/helm/${{ env.OPERATOR_NAME }}
270+
chart-version: ${{ needs.build-container-image.outputs.operator-version }}
271+
app-version: ${{ needs.build-container-image.outputs.operator-version }}
272+
publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }}
273+
helm-version: v3.17.4 # This is currently the latest version which supports pushing to quay.io
274+
244275
openshift-preflight-check:
245276
name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }}
246277
if: |
@@ -259,12 +290,18 @@ jobs:
259290
- arm64
260291
runs-on: ubuntu-latest
261292
steps:
262-
- name: Run OpenShift Preflight Check
263-
uses: stackabletech/actions/run-openshift-preflight@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
293+
- name: Run OpenShift Preflight Check for oci.stackable.tech
294+
uses: stackabletech/actions/run-openshift-preflight@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
264295
with:
265296
image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }}
266297
image-architecture: ${{ matrix.arch }}
267298

299+
- name: Run OpenShift Preflight Check for quay.io
300+
uses: stackabletech/actions/run-openshift-preflight@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
301+
with:
302+
image-index-uri: quay.io/stackable/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }}
303+
image-architecture: ${{ matrix.arch }}
304+
268305
# This job is a required check in GitHub Settings for this repository.
269306
# It saves us having to list many required jobs, or work around dynamically
270307
# named jobs (since there is no concept of required settings).
@@ -300,7 +337,7 @@ jobs:
300337
persist-credentials: false
301338

302339
- name: Send Notification
303-
uses: stackabletech/actions/send-slack-notification@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
340+
uses: stackabletech/actions/send-slack-notification@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
304341
with:
305342
publish-helm-chart-result: ${{ needs.publish-helm-chart.result }}
306343
publish-manifests-result: ${{ needs.publish-index-manifest.result }}

‎.github/workflows/integration-test.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ on:
2323
test:
2424
description: Name of the test. Only used of test-mode is `custom`
2525

26+
permissions: {}
27+
2628
jobs:
2729
test:
2830
name: Run Integration Test
@@ -41,7 +43,7 @@ jobs:
4143
# TODO: Enable the scheduled runs which hard-code what profile to use
4244
- name: Run Integration Test
4345
id: test
44-
uses: stackabletech/actions/run-integration-test@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
46+
uses: stackabletech/actions/run-integration-test@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
4547
with:
4648
replicated-api-token: ${{ secrets.REPLICATED_API_TOKEN }}
4749
test-mode-input: ${{ inputs.test-mode-input }}
@@ -51,7 +53,7 @@ jobs:
5153

5254
- name: Send Notification
5355
if: ${{ failure() || github.run_attempt > 1 }}
54-
uses: stackabletech/actions/send-slack-notification@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
56+
uses: stackabletech/actions/send-slack-notification@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
5557
with:
5658
slack-token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }}
5759
failed-tests: ${{ steps.test.outputs.failed-tests }}

.github/workflows/pr_pre-commit.yaml renamed to .github/workflows/pr_prek.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
---
2-
name: pre-commit
2+
name: prek
33

44
on:
55
pull_request:
66
merge_group:
77

8+
permissions: {}
9+
810
env:
911
CARGO_TERM_COLOR: always
1012
NIX_PKG_MANAGER_VERSION: "2.33.3"
1113
RUST_TOOLCHAIN_VERSION: "nightly-2026-02-24"
1214
HADOLINT_VERSION: "v2.14.0"
13-
PYTHON_VERSION: "3.14"
1415
JINJA2_CLI_VERSION: "1.0.0"
1516

1617
jobs:
17-
pre-commit:
18+
prek:
1819
runs-on: ubuntu-latest
1920
steps:
2021
- name: Install host dependencies
@@ -27,9 +28,8 @@ jobs:
2728
persist-credentials: false
2829
submodules: recursive
2930
fetch-depth: 0
30-
- uses: stackabletech/actions/run-pre-commit@6a84fcfccf4ee37d85217407bc6a94742afbb45f # v0.14.2
31+
- uses: stackabletech/actions/run-prek@a14cbd08d9e034e2361ea9205b32aff0491885db # v0.15.0
3132
with:
32-
python-version: ${{ env.PYTHON_VERSION }}
3333
rust: ${{ env.RUST_TOOLCHAIN_VERSION }}
3434
hadolint: ${{ env.HADOLINT_VERSION }}
3535
nix: ${{ env.NIX_PKG_MANAGER_VERSION }}

‎.pre-commit-config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ repos:
3838
hooks:
3939
- id: shellcheck
4040
args: ["--severity=info"]
41+
# Temporarily needed until all getting_started.sh.j2 scripts are made non-executable.
42+
# As long as they are, prek will treat them as shell scripts and so this hook will fail.
43+
# pre-commit behaves differently apparently and doesn't pass .j2 files to this hook.
44+
exclude: \.j2$
4145

4246
# WARNING (@NickLarsenNZ): Nix users need to install ruff first.
4347
# If you do not, you will need to delete the cached ruff binary shown in the

‎.readme/partials/borrowed/overview_blurb.md.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
<!-- markdownlint-disable MD041 MD051 -->
2-
It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
2+
It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka®, OpenSearch, Trino or Apache Spark™, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
77
### Added
88

99
- BREAKING: Add required CLI argument and env var to set the image repository used to construct final product image names: `IMAGE_REPOSITORY` (`--image-repository`), eg. `oci.example.org/my/namespace` ([#684]).
10+
- Support for Spark `4.1.2` ([#708])
1011

1112
### Fixed
1213

@@ -28,6 +29,7 @@ All notable changes to this project will be documented in this file.
2829
- Internal operator refactoring: introduce dereference() and validate() steps in the reconciler for spark application, spark connect and spark history server([#687]).
2930
- test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#689]).
3031
- Fix the `SparkApplication` CRD description, which incorrectly described it as a "Spark cluster stacklet" rather than a Spark application ([#705]).
32+
- BREAKING: make application templates namespaced instead of cluster wide objects ([#694]).
3133

3234
[#674]: https://github.com/stackabletech/spark-k8s-operator/pull/674
3335
[#679]: https://github.com/stackabletech/spark-k8s-operator/pull/679
@@ -36,8 +38,10 @@ All notable changes to this project will be documented in this file.
3638
[#687]: https://github.com/stackabletech/spark-k8s-operator/pull/687
3739
[#689]: https://github.com/stackabletech/spark-k8s-operator/pull/689
3840
[#692]: https://github.com/stackabletech/spark-k8s-operator/pull/692
41+
[#694]: https://github.com/stackabletech/spark-k8s-operator/pull/694
3942
[#696]: https://github.com/stackabletech/spark-k8s-operator/pull/696
4043
[#705]: https://github.com/stackabletech/spark-k8s-operator/pull/705
44+
[#708]: https://github.com/stackabletech/spark-k8s-operator/pull/708
4145

4246
## [26.3.0] - 2026-03-16
4347

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
This is a Kubernetes operator to manage [Apache Spark](https://spark.apache.org/) jobs.
1616

1717
<!-- markdownlint-disable MD041 MD051 -->
18-
It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka, Apache Druid, Trino or Apache Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
18+
It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Apache Kafka®, OpenSearch, Trino or Apache Spark™, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud](#supported-platforms).
1919

2020
## Installation
2121

‎deny.toml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,25 @@ ignore = [
4545
# This version is pulled in transitively via num-bigint-dig -> rsa -> stackable-certs and cannot be
4646
# updated until the upstream rsa crate bumps its rand dependency.
4747
"RUSTSEC-2026-0097",
48+
49+
# https://rustsec.org/advisories/RUSTSEC-2026-0173
50+
# The author of `proc-macro-error2` has [confirmed](https://github.com/GnomedDev/proc-macro-error-2/issues/17#issuecomment-4643215473)
51+
# that the crate is no longer maintained and recommends that users migrate away from it.
52+
#
53+
# There currently is no way for us to negate this advisory, because that crate is not used
54+
# directly by us. We need to wait for new versions of oci-spec and getset. See the following
55+
# issue which tracks moving to a newer getset version: https://github.com/youki-dev/oci-spec-rs/issues/340
56+
#
57+
# proc-macro-error2 v2.0.1
58+
# └── getset v0.1.6
59+
# └── oci-spec v0.9.0
60+
# └── boil v0.2.1
61+
#
62+
# Alternate crates are:
63+
#
64+
# - https://crates.io/crates/manyhow
65+
# - https://github.com/SergioBenitez/proc-macro2-diagnostics
66+
"RUSTSEC-2026-0173",
4867
]
4968

5069
[bans]

‎deploy/helm/spark-k8s-operator/values/quay.io.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
# Values overlay for chart packages published to quay.io.
3+
image:
4+
repository: quay.io/stackable/sdp

‎docs/modules/spark-k8s/examples/example-history-app.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ metadata:
55
name: spark-pi-s3-1
66
spec:
77
sparkImage:
8-
productVersion: 4.1.1
8+
productVersion: 4.1.2
99
pullPolicy: IfNotPresent
1010
mode: cluster
1111
mainClass: org.apache.spark.examples.SparkPi

0 commit comments

Comments
 (0)