Merge branch 'main' into airflow-3.2.1

Author: NickLarsenNZ

.scripts/upload_new_boost_version.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# This script mirrors the boostorg/boost source bundle for the given version to Nexus.
+# The boost source bundle is architecture independent.
+# It contains its own build system (b2) which is also built from source before building boost itself, so we don't need to worry about architecture specific builds.
+# This artifact is used by the hadoop/boost local image.
+
+
+set -euo pipefail
+
+VERSION=${1:?"Missing version number argument (arg 1)"}
+NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
+
+read -r -s -p "Nexus Password: " NEXUS_PASSWORD
+echo ""
+
+# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory
+# Find the directory name of the script
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# the temp directory used, within $DIR
+WORK_DIR=$(mktemp -d -p "$DIR")
+
+# check if tmp dir was created
+if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+  echo "Could not create temp dir"
+  exit 1
+fi
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$WORK_DIR"
+}
+
+# register the cleanup function to be called on the EXIT signal
+trap cleanup EXIT
+
+cd "$WORK_DIR" || exit
+
+# boost does not currently publish signatures or SBOMs
+BOOST_UNDERSCORE="$(echo "${VERSION}" | tr '.' '_')"
+BOOST_TARBALL="boost_${BOOST_UNDERSCORE}.tar.bz2"
+DOWNLOAD_URL="https://archives.boost.io/release/$VERSION/source/$BOOST_TARBALL"
+
+echo "Downloading boost"
+if ! curl --fail -Ls -O "$DOWNLOAD_URL"; then
+  echo "Failed to download from $DOWNLOAD_URL"
+  exit 1
+fi
+
+FILE_NAME=$(basename "$DOWNLOAD_URL")
+
+echo "Uploading boost to Nexus"
+if ! curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/boost/'; then
+  echo "Failed to upload boost to Nexus"
+  exit 1
+fi
+
+echo "Successfully uploaded new version of boost ($VERSION) to Nexus"
+echo "https://repo.stackable.tech/service/rest/repository/browse/packages/boost/"

CHANGELOG.md

@@ -14,9 +14,10 @@ All notable changes to this project will be documented in this file.
 - opa: Add `1.16.2` ([#1509]).
 - kafka: Add `3.9.2` and `4.2.1` ([#1483]).
 - trino: Add `481` ([#1518]).
+- superset: Add `6.1.0` ([#1514]).
+- zookeeper: Add `3.9.5` ([#1515]).
+- hadoop: Add `3.5.0` and `3.4.3` ([#1511]).
 - airflow: Add `3.2.1` ([#1519]).
-- superset: Add `6.1.0` ([#1514])
-- zookeeper: Add `3.9.5` ([#1515])
 
 ### Changed
 
@@ -27,6 +28,7 @@ All notable changes to this project will be documented in this file.
 - statsd_exporter: Bump version from `0.28.0` to `0.30.0` ([#1524]).
   This uses a git mirror and patchable instead of sourcing from Nexus.
 - superset: Bump statsd_exporter to `0.30.0` ([#1524]).
+- hadoop: Refactor cloud library provisioning to downstream images. The jars are now conveniently placed in `/stackable/hadoop-cloud-libraries/` to be easily picked up ([#1511]).
 
 ### Fixed
 
@@ -41,6 +43,7 @@ All notable changes to this project will be documented in this file.
 
 - nifi: Remove `1.28.1`, deprecate `2.7.2` ([#1520]).
 - opa: Remove `1.8.0` ([#1509]).
+- spark-k8s: Remove `3.5.7` and `4.0.1` ([#1525]).
 
 [#1446]: https://github.com/stackabletech/docker-images/pull/1446
 [#1452]: https://github.com/stackabletech/docker-images/pull/1452
@@ -57,13 +60,15 @@ All notable changes to this project will be documented in this file.
 [#1507]: https://github.com/stackabletech/docker-images/pull/1507
 [#1509]: https://github.com/stackabletech/docker-images/pull/1509
 [#1510]: https://github.com/stackabletech/docker-images/pull/1510
+[#1511]: https://github.com/stackabletech/docker-images/pull/1511
 [#1512]: https://github.com/stackabletech/docker-images/pull/1512
 [#1514]: https://github.com/stackabletech/docker-images/pull/1514
 [#1515]: https://github.com/stackabletech/docker-images/pull/1515
 [#1518]: https://github.com/stackabletech/docker-images/pull/1518
 [#1519]: https://github.com/stackabletech/docker-images/pull/1519
 [#1520]: https://github.com/stackabletech/docker-images/pull/1520
 [#1521]: https://github.com/stackabletech/docker-images/pull/1521
+[#1525]: https://github.com/stackabletech/docker-images/pull/1525
 [#1524]: https://github.com/stackabletech/docker-images/pull/1524
 
 ## [26.3.0] - 2026-03-16

Cargo.lock

@@ -14,6 +14,57 @@ targets = [
 
 [advisories]
 yanked = "deny"
+ignore = [
+  # https://rustsec.org/advisories/RUSTSEC-2023-0071
+  # "rsa" crate: Marvin Attack: potential key recovery through timing sidechannel
+  #
+  # No patch is yet available, however work is underway to migrate to a fully constant-time implementation.
+  # So we need to accept this, as of SDP 26.3 we are "only" using the crate to create private +
+  # public key pairs used by webhooks, such as conversion or mutating webhooks.
+  #
+  # https://github.com/RustCrypto/RSA/issues/19 is the tracking issue
+  "RUSTSEC-2023-0071",
+
+  # https://rustsec.org/advisories/RUSTSEC-2024-0436
+  # The "paste" crate is no longer maintained because the owner states that the implementation is
+  # finished. There are at least two (forked) alternatives which state to be maintained. They'd
+  # need to be vetted before a potential switch. Additionally, they'd need to be in a maintained
+  # state for a couple of years to provide any benefit over using "paste".
+  #
+  # This crate is only used in a single place in the xtask package inside the declarative
+  # "write_crd" macro. The impact of vulnerabilities, if any, should be fairly minimal.
+  #
+  # See thread: https://users.rust-lang.org/t/paste-alternatives/126787/4
+  #
+  # This can only be removed again if we decide to use a different crate.
+  "RUSTSEC-2024-0436",
+
+  # https://rustsec.org/advisories/RUSTSEC-2026-0097
+  # rand 0.8.5 is unsound when log+thread_rng features are enabled and a custom logger calls rand::rng().
+  #
+  # This version is pulled in transitively via num-bigint-dig -> rsa -> stackable-certs and cannot be
+  # updated until the upstream rsa crate bumps its rand dependency.
+  "RUSTSEC-2026-0097",
+
+  # https://rustsec.org/advisories/RUSTSEC-2026-0173
+  # The author of `proc-macro-error2` has [confirmed](https://github.com/GnomedDev/proc-macro-error-2/issues/17#issuecomment-4643215473)
+  # that the crate is no longer maintained and recommends that users migrate away from it.
+  #
+  # There currently is no way for us to negate this advisory, because that crate is not used
+  # directly by us. We need to wait for new versions of oci-spec and getset. See the following
+  # issue which tracks moving to a newer getset version: https://github.com/youki-dev/oci-spec-rs/issues/340
+  #
+  # proc-macro-error2 v2.0.1
+  #  └── getset v0.1.6
+  #      └── oci-spec v0.9.0
+  #          └── boil v0.2.1
+  #
+  # Alternate crates are:
+  #
+  # - https://crates.io/crates/manyhow
+  # - https://github.com/SergioBenitez/proc-macro2-diagnostics
+  "RUSTSEC-2026-0173",
+]
 
 [bans]
 multiple-versions = "allow"
@@ -31,7 +82,7 @@ allow = [
   "LicenseRef-webpki",
   "MIT",
   "MPL-2.0",
-  "OpenSSL",           # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details
+  "OpenSSL", # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details
   "Unicode-3.0",
   "Unicode-DFS-2016",
   "Zlib",
@@ -52,6 +103,7 @@ license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]
 [sources]
 unknown-registry = "deny"
 unknown-git = "deny"
+allow-git = ["https://github.com/kube-rs/kube-rs"]
 
 [sources.allow-org]
 github = ["stackabletech"]

deny.toml

@@ -15,7 +15,7 @@ authorizer-version = "0.7.0"
 # https://druid.apache.org/docs/34.0.0/operations/java/
 java-base = "17"
 java-devel = "17"
-"hadoop/hadoop" = "3.4.2"
+"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3
 
 # Deprecated since 26.3
 [versions."34.0.0".build-arguments]
@@ -26,7 +26,7 @@ authorizer-version = "0.7.0"
 # https://druid.apache.org/docs/35.0.1/operations/java/
 java-base = "21"
 java-devel = "21"
-"hadoop/hadoop" = "3.4.2"
+"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3
 
 [versions."35.0.1".build-arguments]
 authorizer-version = "0.7.0"

druid/boil-config.toml

@@ -20,4 +20,24 @@ java-devel = "11"
 [versions."3.4.2".build-arguments]
 async-profiler-version = "2.9"
 jmx-exporter-version = "1.3.0"
-hdfs-utils-version = "0.5.0"
+hdfs-utils-version = "0.6.0"
+
+[versions."3.4.3".local-images]
+"hadoop/hadoop" = "3.4.3"
+java-base = "11"
+java-devel = "11"
+
+[versions."3.4.3".build-arguments]
+async-profiler-version = "2.9"
+jmx-exporter-version = "1.3.0"
+hdfs-utils-version = "0.6.0"
+
+[versions."3.5.0".local-images]
+"hadoop/hadoop" = "3.5.0"
+java-base = "17"
+java-devel = "17"
+
+[versions."3.5.0".build-arguments]
+async-profiler-version = "2.9"
+jmx-exporter-version = "1.3.0"
+hdfs-utils-version = "0.6.0"

hadoop/boil-config.toml

@@ -1,11 +1,15 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
+FROM local-image/shared/boost AS boost-builder
 FROM local-image/java-devel AS hadoop-builder
 
 ARG PRODUCT_VERSION
 ARG RELEASE_VERSION
 ARG PROTOBUF_VERSION
+ARG AWS_JAVA_SDK_BUNDLE_VERSION
+ARG AZURE_STORAGE_VERSION
+ARG AZURE_KEYVAULT_CORE_VERSION
 ARG STACKABLE_USER_UID
 
 WORKDIR /stackable
@@ -14,17 +18,22 @@ COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable
 COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF_VERSION} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF_VERSION}
 
 RUN <<EOF
-rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
 microdnf update
-# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
-# automake and libtool are required to build protobuf
-microdnf install boost1.78-devel automake libtool
+# libstdc++ is a runtime dependency for boost,
+# automake and libtool are needed to build protobuf
+microdnf install libstdc++ automake libtool
 microdnf clean all
 rm -rf /var/cache/yum
 mkdir /opt/protobuf
 chown ${STACKABLE_USER_UID}:0 /opt/protobuf
 EOF
 
+COPY --chown=${STACKABLE_USER_UID}:0 --from=boost-builder /stackable/boost /stackable/boost
+
+ENV BOOST_ROOT=/stackable/boost
+ENV LD_LIBRARY_PATH=/stackable/boost/lib
+ENV CPATH=/stackable/boost/include
+
 USER ${STACKABLE_USER_UID}
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
@@ -35,10 +44,18 @@ RUN <<EOF
     # Create snapshot of the source code including custom patches
     tar -czf /stackable/protobuf-${PROTOBUF_VERSION}-src.tar.gz .
 
-    ./autogen.sh
-    ./configure --prefix=/opt/protobuf
-    make "-j$(nproc)"
-    make install
+    if [ "$PROTOBUF_VERSION" == "3.7.1" ] || [ "$PROTOBUF_VERSION" == "3.21.12" ]; then
+        ./autogen.sh
+        ./configure --prefix=/opt/protobuf
+        make "-j$(nproc)"
+        make install
+    else
+        # protobuf > 3.21 bundles abseil-cpp and utf8_range as git submodules
+        git submodule update --init --recursive
+        cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
+        cmake --build build --parallel "$(nproc)"
+        cmake --install build --prefix /opt/protobuf
+    fi
     (cd .. && rm -r ${PROTOBUF_VERSION})
 EOF
 
@@ -126,3 +143,31 @@ rm -rf /stackable/.m2
 # Set correct groups; make sure only required artifacts for the final image are located in /stackable
 chmod -R g=u /stackable
 EOF
+
+RUN <<EOF
+NEW_VERSION=${PRODUCT_VERSION}-stackable${RELEASE_VERSION}
+
+# Place cloud libraries (for s3a:// and abfs://) in the /stackable/hadoop-cloud-libraries/ folder to
+# have one central place for dependant images to pull them from
+mkdir -p /stackable/hadoop-cloud-libraries/
+
+if [[ "$PRODUCT_VERSION" == "3.3.6" || "$PRODUCT_VERSION" == "3.4.2" ]]; then
+    cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    # For some reason it was renamed from "aws-java-sdk-bundle" to "bundle"
+    if [[ "$PRODUCT_VERSION" == "3.3.6" ]]; then
+        cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar
+    else
+        cp /stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar
+    fi
+    cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+else
+    cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more.
+    curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
+    cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+    cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
+fi
+EOF

hadoop/hadoop/Dockerfile

@@ -1,11 +1,44 @@
 [versions."3.3.6".local-images]
 java-devel = "11"
+"shared/boost" = "1.72.0" # I could not find a documented recommended version
 
 [versions."3.3.6".build-arguments]
-protobuf-version = "3.7.1"
+protobuf-version = "3.7.1" # https://github.com/apache/hadoop/blob/rel/release-3.3.6/BUILDING.txt
+aws-java-sdk-bundle-version = "1.12.367"  # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+
 
 [versions."3.4.2".local-images]
-java-devel = "11"
+java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
 
 [versions."3.4.2".build-arguments]
-protobuf-version = "3.7.1"
+protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
+aws-java-sdk-bundle-version = "2.29.52"   # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.2
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.2
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+
+
+
+[versions."3.4.3".local-images]
+java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
+
+[versions."3.4.3".build-arguments]
+protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
+aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+
+
+
+[versions."3.5.0".local-images]
+java-devel = "17" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.86.0" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt
+
+[versions."3.5.0".build-arguments]
+protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt
+aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.5.0
+azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.5.0
+azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1