Merge remote-tracking branch 'origin/main' into chore/hive-26.7

sbernauer · sbernauer · commit 09b72b2e75a8 · 2026-06-18T14:26:10.000+02:00
diff --git a/.github/ISSUE_TEMPLATE/update-product-druid.md b/.github/ISSUE_TEMPLATE/update-product-druid.md
@@ -31,6 +31,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Release a new version of [druid-opa-authorizer](https://github.com/stackabletech/druid-opa-authorizer), if compiling against the new Druid version(s) requires changes in the code.
 - [ ] Update `boil-config.toml` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions).
 - [ ] Update `boil-config.toml` to the latest supported version of JVM (base and devel).
+- [ ] Update `boil-config.toml` to the latest supported version of Hadoop.
 - [ ] Initialize new product versions with patchable and add patches if applicable.
 - [ ] Delete old patch directories.
 - [ ] Check the corresponding operator (getting_started / kuttl / supported-versions) for usage of the versions.
diff --git a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md
@@ -32,6 +32,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Update `boil-config.toml` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions).
 - [ ] Upload new versions (see the `hbase/*.sh` scripts).
 - [ ] Update `boil-config.toml` to the latest supported version of JVM (base and devel).
+- [ ] Update `boil-config.toml` to the latest supported version of Hadoop.
 - [ ] Update other dependencies if applicable (eg: phoenix, opa_authorizer, etc).
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR to the list below.
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-hive.md b/.github/ISSUE_TEMPLATE/update-product-hive.md
@@ -29,6 +29,7 @@ Add/Change/Remove anything that isn't applicable anymore
 
 - [ ] Update `boil-config.toml` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions).
 - [ ] Update `boil-config.toml` to the latest supported version of JVM (base and devel).
+- [ ] Update `boil-config.toml` to the latest supported version of Hadoop.
 - [ ] Update other dependencies if applicable (eg: jmx_exporter, aws_java_sdk_bundle, etc).
 - [ ] Initialize new product versions with patchable and add patches if applicable.
 - [ ] Delete old patch directories.
diff --git a/.github/ISSUE_TEMPLATE/update-product-spark.md b/.github/ISSUE_TEMPLATE/update-product-spark.md
@@ -30,6 +30,7 @@ Add/Change/Remove anything that isn't applicable anymore
 - [ ] Update `boil-config.toml` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions).
 - [ ] Upload new version (see `spark/upload_new_spark_version.sh`).
 - [ ] Update `boil-config.toml` to the latest supported version of JVM (base and devel).
+- [ ] Update `boil-config.toml` to the latest supported version of Hadoop.
 - [ ] Ensure that the transitive dependencies in `boil-config.toml` are compatible with the Spark version(s).
 - [ ] Update other dependencies if applicable (eg: python, jmx_exporter, aws_java_sdk_bundle, etc).
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@ All notable changes to this project will be documented in this file.
   This uses a git mirror and patchable instead of sourcing from Nexus.
 - superset: Bump statsd_exporter to `0.30.0` ([#1524]).
 - hadoop: Refactor cloud library provisioning to downstream images. The jars are now conveniently placed in `/stackable/hadoop-cloud-libraries/` to be easily picked up ([#1511]).
+- spark: bump hadoop `3.4.2` to `3.4.3` ([#1533])
 - hive: Bump `4.2.0` to Hadoop `3.4.3` ([#1539]).
 
 ### Fixed
@@ -72,7 +73,11 @@ All notable changes to this project will be documented in this file.
 [#1521]: https://github.com/stackabletech/docker-images/pull/1521
 [#1524]: https://github.com/stackabletech/docker-images/pull/1524
 [#1525]: https://github.com/stackabletech/docker-images/pull/1525
+<<<<<<< HEAD
 [#1539]: https://github.com/stackabletech/docker-images/pull/1539
+=======
+[#1533]: https://github.com/stackabletech/docker-images/pull/1533
+>>>>>>> origin/main
 
 ## [26.3.0] - 2026-03-16
 
diff --git a/druid/Dockerfile b/druid/Dockerfile
@@ -145,7 +145,8 @@ COPY --chown=${STACKABLE_USER_UID}:0 druid/licenses /licenses
 RUN <<EOF
 microdnf update
 microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt
+# List all rpm packages installed in the system
+rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile
@@ -110,7 +110,8 @@ microdnf install \
   fuse-libs \
   tar
 microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt
+# List all rpm packages installed in the system
+rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile
@@ -10,6 +10,7 @@ ARG PROTOBUF_VERSION
 ARG AWS_JAVA_SDK_BUNDLE_VERSION
 ARG AZURE_STORAGE_VERSION
 ARG AZURE_KEYVAULT_CORE_VERSION
+ARG ANALYTICSACCELERATOR_S3_VERSION
 ARG STACKABLE_USER_UID
 
 WORKDIR /stackable
@@ -166,6 +167,8 @@ else
     cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
     # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more.
     curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
+    # Starting with Hadoop 3.4.3 the S3A InputStreamType factory references analyticsaccelerator-s3 (software.amazon.s3.analyticsaccelerator.request.ObjectClient) during S3AFileSystem.initialize.
+    curl -o /stackable/hadoop-cloud-libraries/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar"
     cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
     cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
     cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
diff --git a/hadoop/hadoop/boil-config.toml b/hadoop/hadoop/boil-config.toml
@@ -30,6 +30,7 @@ protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release
 aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
 azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
 azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+analyticsaccelerator-s3-version = "1.3.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
 
 
 
@@ -42,3 +43,4 @@ protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release-
 aws-java-sdk-bundle-version = "2.35.4"    # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.5.0
 azure-storage-version = "7.0.1"           # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.5.0
 azure-keyvault-core-version = "1.0.0"     # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+analyticsaccelerator-s3-version = "1.3.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
diff --git a/hbase/Dockerfile b/hbase/Dockerfile
@@ -120,7 +120,8 @@ microdnf install \
   zip
 
 microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt
+# List all rpm packages installed in the system
+rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
diff --git a/hive/Dockerfile b/hive/Dockerfile
@@ -175,7 +175,8 @@ COPY hive/licenses /licenses
 RUN <<EOF
 microdnf update
 microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt
+# List all rpm packages installed in the system
+rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
diff --git a/kafka/Dockerfile b/kafka/Dockerfile
@@ -100,7 +100,8 @@ microdnf install \
   cyrus-sasl-gssapi
 
 microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt
+# List all rpm packages installed in the system
+rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
diff --git a/rust/boil/README.md b/rust/boil/README.md
@@ -31,3 +31,42 @@ boil image list
 # Display a list of versions of the image located in the 'airflow' folder
 boil image list airflow
 ```
+
+## Configuration
+
+### Global Configuration
+
+Project-wide options are configured in the global `boil.toml` configuration file, which is located
+at the root of the repository. The config currently supports setting global build arguments and
+metadata used for image annotations and tag construction.
+
+```toml
+[build-arguments]
+FOO = "bar"                                # Available in all image builds
+
+[metadata]
+documentation = "https://docs.example.org" # URL to the documentation for images
+source = "https://source.example.org"      # URL to the source of the images, like a Git forge
+authors = "Foo <foo@example.org>"          # One or more source authors
+vendor-tag-prefix = "foo"                  # Prefix used in tag construction, eg. `1.2.3-foo4.5.6`
+vendor = "Foo"                             # Name of the vendor
+licenses = "Apache-2.0"                    # One or more licenses, space separated
+```
+
+### Per-image Configuration
+
+boil discovers images by globbing for `boil-config.toml` files in sub directories of the repository.
+These configuration files set image specific options, like build arguments, references to local
+images, and registry metadata.
+
+```toml
+[metadata.registries]
+"oci.example.org" = { namespace = "my/namespace" } # Used for image checks
+
+[versions."1.2.3".local-images]                    # Specify references to local images per version
+foo = "1.2.3"
+bar = "4.5.6"
+
+[versions."1.2.3".build-arguments]                 # Specify build arguments per version
+FOO = "bar"
+```
diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml
@@ -6,7 +6,7 @@
 containerfile = "Dockerfile.3"
 
 [versions."3.5.8".local-images]
-"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3
+"hadoop/hadoop" = "3.4.3"
 java-base = "17"
 java-devel = "17"
 hbase = "2.6.3"
@@ -22,19 +22,20 @@ tini-version = "0.19.0"
 hbase-connector-version = "1.0.1_3.5.8"
 
 
+
 # LTS since 26.7.0
 [versions."4.1.1"]
 containerfile = "Dockerfile.4"
 
 [versions."4.1.1".local-images]
-"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3
+"hadoop/hadoop" = "3.4.3"
 java-base = "21"
 java-devel = "21"
 
 [versions."4.1.1".build-arguments]
 python-version = "3.12"
-jackson-dataformat-xml-version = "2.15.2" # Needs to match https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7
-stax2-api-version = "4.2.1"               # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-woodstox-core-version = "6.5.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+jackson-dataformat-xml-version = "2.20.0" # Needs to match https://mvnrepository.com/artifact/org.apache.spark/spark-parent_2.13/4.1.1
+stax2-api-version = "4.2.2"               # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0
+woodstox-core-version = "7.1.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
diff --git a/zookeeper/Dockerfile b/zookeeper/Dockerfile
@@ -97,7 +97,8 @@ COPY zookeeper/licenses /licenses
 RUN <<EOF
 microdnf update
 microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt
+# List all rpm packages installed in the system
+rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
 chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
