feat: Add Hadoop 3.5.0

Author: sbernauer

.scripts/upload_new_boost_version.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# This script mirrors the boostorg/boost source bundle for the given version to Nexus.
+# The boost source bundle is architecture independent.
+# It contains its own build system (b2) which is also built from source before building boost itself, so we don't need to worry about architecture specific builds.
+# This artifact is used by the hadoop/boost local image.
+
+
+set -euo pipefail
+
+VERSION=${1:?"Missing version number argument (arg 1)"}
+NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
+
+read -r -s -p "Nexus Password: " NEXUS_PASSWORD
+echo ""
+
+# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory
+# Find the directory name of the script
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# the temp directory used, within $DIR
+WORK_DIR=$(mktemp -d -p "$DIR")
+
+# check if tmp dir was created
+if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+  echo "Could not create temp dir"
+  exit 1
+fi
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$WORK_DIR"
+}
+
+# register the cleanup function to be called on the EXIT signal
+trap cleanup EXIT
+
+cd "$WORK_DIR" || exit
+
+# boost does not currently publish signatures or SBOMs
+BOOST_UNDERSCORE="$(echo "${VERSION}" | tr '.' '_')"
+BOOST_TARBALL="boost_${BOOST_UNDERSCORE}.tar.bz2"
+DOWNLOAD_URL="https://archives.boost.io/release/$VERSION/source/$BOOST_TARBALL"
+
+echo "Downloading boost"
+if ! curl --fail -Ls -O "$DOWNLOAD_URL"; then
+  echo "Failed to download from $DOWNLOAD_URL"
+  exit 1
+fi
+
+FILE_NAME=$(basename "$DOWNLOAD_URL")
+
+echo "Uploading boost to Nexus"
+if ! curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/boost/'; then
+  echo "Failed to upload boost to Nexus"
+  exit 1
+fi
+
+echo "Successfully uploaded new version of boost ($VERSION) to Nexus"
+echo "https://repo.stackable.tech/service/rest/repository/browse/packages/boost/"

hadoop/boil-config.toml

@@ -20,4 +20,24 @@ java-devel = "11"
 [versions."3.4.2".build-arguments]
 async-profiler-version = "2.9"
 jmx-exporter-version = "1.3.0"
-hdfs-utils-version = "0.5.0"
+hdfs-utils-version = "0.6.0"
+
+[versions."3.4.3".local-images]
+"hadoop/hadoop" = "3.4.3"
+java-base = "11"
+java-devel = "11"
+
+[versions."3.4.3".build-arguments]
+async-profiler-version = "2.9"
+jmx-exporter-version = "1.3.0"
+hdfs-utils-version = "0.6.0"
+
+[versions."3.5.0".local-images]
+"hadoop/hadoop" = "3.5.0"
+java-base = "17"
+java-devel = "17"
+
+[versions."3.5.0".build-arguments]
+async-profiler-version = "2.9"
+jmx-exporter-version = "1.3.0"
+hdfs-utils-version = "0.6.0"

hadoop/hadoop/Dockerfile

@@ -1,6 +1,7 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
+FROM local-image/shared/boost AS boost-builder
 FROM local-image/java-devel AS hadoop-builder
 
 ARG PRODUCT_VERSION
@@ -14,17 +15,22 @@ COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable
 COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF_VERSION} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF_VERSION}
 
 RUN <<EOF
-rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
 microdnf update
-# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code
-# automake and libtool are required to build protobuf
-microdnf install boost1.78-devel automake libtool
+# libstdc++ is a runtime dependency for boost,
+# automake and libtool are needed to build protobuf
+microdnf install libstdc++ automake libtool
 microdnf clean all
 rm -rf /var/cache/yum
 mkdir /opt/protobuf
 chown ${STACKABLE_USER_UID}:0 /opt/protobuf
 EOF
 
+COPY --chown=${STACKABLE_USER_UID}:0 --from=boost-builder /stackable/boost /stackable/boost
+
+ENV BOOST_ROOT=/stackable/boost
+ENV LD_LIBRARY_PATH=/stackable/boost/lib
+ENV CPATH=/stackable/boost/include
+
 USER ${STACKABLE_USER_UID}
 # This Protobuf version is the exact version as used in the Hadoop Dockerfile
 # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh
@@ -35,10 +41,18 @@ RUN <<EOF
     # Create snapshot of the source code including custom patches
     tar -czf /stackable/protobuf-${PROTOBUF_VERSION}-src.tar.gz .
 
-    ./autogen.sh
-    ./configure --prefix=/opt/protobuf
-    make "-j$(nproc)"
-    make install
+    if [ "$PROTOBUF_VERSION" == "3.7.1" ] || [ "$PROTOBUF_VERSION" == "3.21.12" ]; then
+        ./autogen.sh
+        ./configure --prefix=/opt/protobuf
+        make "-j$(nproc)"
+        make install
+    else
+        # protobuf > 3.21 bundles abseil-cpp and utf8_range as git submodules
+        git submodule update --init --recursive
+        cmake -S . -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF
+        cmake --build build --parallel $(nproc)
+        cmake --install build --prefix /opt/protobuf
+    fi
     (cd .. && rm -r ${PROTOBUF_VERSION})
 EOF
 

hadoop/hadoop/boil-config.toml

@@ -1,11 +1,30 @@
 [versions."3.3.6".local-images]
 java-devel = "11"
+"shared/boost" = "1.72.0" # I could not find a documented recommended version
 
 [versions."3.3.6".build-arguments]
-protobuf-version = "3.7.1"
+protobuf-version = "3.7.1" # https://github.com/apache/hadoop/blob/rel/release-3.3.6/BUILDING.txt
+
 
 [versions."3.4.2".local-images]
-java-devel = "11"
+java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
 
 [versions."3.4.2".build-arguments]
-protobuf-version = "3.7.1"
+protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
+
+
+[versions."3.4.3".local-images]
+java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
+
+[versions."3.4.3".build-arguments]
+protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
+
+
+[versions."3.5.0".local-images]
+java-devel = "17" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+"shared/boost" = "1.86.0" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt
+
+[versions."3.5.0".build-arguments]
+protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt

hadoop/hadoop/stackable/patches/3.4.3/0001-YARN-11527-Update-node.js.patch

@@ -0,0 +1,22 @@
+From 7ad31a922a9fbcecd884b4bdf5c416f6b0ea539e Mon Sep 17 00:00:00 2001
+From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
+Date: Tue, 26 May 2026 15:40:40 +0200
+Subject: YARN-11527-Update-node.js
+
+---
+ hadoop-project/pom.xml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
+index 0813904f98a..f837b1f5201 100644
+--- a/hadoop-project/pom.xml
++++ b/hadoop-project/pom.xml
+@@ -236,7 +236,7 @@
+     <woodstox.version>5.4.0</woodstox.version>
+     <nimbus-jose-jwt.version>10.4</nimbus-jose-jwt.version>
+     <jcip-annotations.version>1.0-1</jcip-annotations.version>
+-    <nodejs.version>v12.22.1</nodejs.version>
++    <nodejs.version>v14.17.0</nodejs.version>
+     <yarnpkg.version>v1.22.5</yarnpkg.version>
+     <apache-ant.version>1.10.13</apache-ant.version>
+     <jmh.version>1.20</jmh.version>