chore(spark/hbase-connectors): bump hbase to 2.6.6 (#1573)

* chore: build spark/hbase-connectors against hbase lts

* fix: add generated protobuf classes to the image

* docs: update changelog

* better changelog

* more bumps

Author: razvan

CHANGELOG.md

@@ -26,6 +26,7 @@ All notable changes to this project will be documented in this file.
 - spark: Add `4.1.2` ([#1550]).
 - opensearch: Add `3.6.0` ([#1549]).
 - opensearch-dashboards: Add `3.6.0` ([#1551]).
+- spark: Add hbase-connector protobuf classes ([#1573]).
 
 ### Changed
 
@@ -42,6 +43,7 @@ All notable changes to this project will be documented in this file.
 - hive: Bump `4.2.0` to Hadoop `3.4.3` ([#1539]).
 - java-devel: Bump Maven to 3.9.16 ([#1548]).
 - spark: fix graceful shutdown at SIGTERM ([#1564])
+- spark-k8s/hbase-connectors: bump hbase to 2.6.6 ([#1573]).
 
 ### Fixed
 

spark-k8s/boil-config.toml

@@ -9,7 +9,7 @@ containerfile = "Dockerfile.3"
 "hadoop/hadoop" = "3.4.3"
 java-base = "17"
 java-devel = "17"
-hbase = "2.6.4"
+hbase = "2.6.6"
 "spark-k8s/hbase-connectors" = "1.0.1_3.5.8"
 
 [versions."3.5.8".build-arguments]

spark-k8s/hbase-connectors/Dockerfile

@@ -54,6 +54,9 @@ ARG PRODUCT_VERSION
 ARG RELEASE_VERSION
 ARG HADOOP_VERSION
 ARG HBASE_VERSION
+ARG HBASE_THIRDPARTY_VERSION
+ARG PROTOBUF_VERSION
+ARG PROTOBUF_PLUGIN_VERSION
 ARG STACKABLE_USER_UID
 
 # Patch the hbase-connectors source code
@@ -107,12 +110,18 @@ mvn \
     --define scala.binary.version="${SCALA_BINARY_VERSION}" \
     --define hadoop-three.version="${HADOOP_VERSION}" \
     --define hbase.version="${HBASE_VERSION}" \
+    --define hbase-thirdparty.version="${HBASE_THIRDPARTY_VERSION}" \
+    --define external.protobuf.version="${PROTOBUF_VERSION}" \
+    --define protobuf.plugin.version="${PROTOBUF_PLUGIN_VERSION}" \
     --define skipTests \
     --define maven.test.skip=true \
     clean package
 
 
 cp "$(pwd)/hbase-spark/target/hbase-spark-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-${NEW_VERSION}.jar
+# The hbase-spark JAR is a plain module JAR and does not bundle its
+# dependencies. The generated protobuf classes live here.
+cp "$(pwd)/hbase-spark-protocol-shaded/target/hbase-spark-protocol-shaded-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-protocol-shaded-${NEW_VERSION}.jar
 
 chmod g=u /stackable/hbase-connector-${NEW_VERSION}-src.tar.gz .
 EOF

spark-k8s/hbase-connectors/boil-config.toml

@@ -5,4 +5,12 @@ java-devel = "17"
 spark-version = "3.5.8"
 # TODO: Bump to 3.4.3. Note that this causes compilation errors that need to be fixed.
 hadoop-version = "3.4.2"
-hbase-version = "2.6.3"
+hbase-version = "2.6.6"
+# The following three must match the hbase-thirdparty (and its bundled protobuf)
+# shipped by the hbase-shaded-client in the final Spark image. That client comes
+# from the `hbase` local-image in spark-k8s/boil-config.toml (currently 2.6.6),
+# whose hbase-thirdparty is 4.1.13 -> protobuf 4.34.0. Keep these in sync if the
+# Spark image's HBASE_VERSION changes.
+hbase-thirdparty-version = "4.1.13"
+protobuf-version = "4.34.0"
+protobuf-plugin-version = "0.6.1"