diff --git a/CHANGELOG.md b/CHANGELOG.md index 2999a4f99..a728ee162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ All notable changes to this project will be documented in this file. - spark: Add `4.1.2` ([#1550]). - opensearch: Add `3.6.0` ([#1549]). - opensearch-dashboards: Add `3.6.0` ([#1551]). +- spark: Add hbase-connector protobuf classes ([#1573]). ### Changed @@ -42,6 +43,7 @@ All notable changes to this project will be documented in this file. - hive: Bump `4.2.0` to Hadoop `3.4.3` ([#1539]). - java-devel: Bump Maven to 3.9.16 ([#1548]). - spark: fix graceful shutdown at SIGTERM ([#1564]) +- spark-k8s/hbase-connectors: bump hbase to 2.6.6 ([#1573]). ### Fixed diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 724c436d9..a6e78665a 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -9,7 +9,7 @@ containerfile = "Dockerfile.3" "hadoop/hadoop" = "3.4.3" java-base = "17" java-devel = "17" -hbase = "2.6.4" +hbase = "2.6.6" "spark-k8s/hbase-connectors" = "1.0.1_3.5.8" [versions."3.5.8".build-arguments] diff --git a/spark-k8s/hbase-connectors/Dockerfile b/spark-k8s/hbase-connectors/Dockerfile index e9821059f..2e11e56c5 100644 --- a/spark-k8s/hbase-connectors/Dockerfile +++ b/spark-k8s/hbase-connectors/Dockerfile @@ -54,6 +54,9 @@ ARG PRODUCT_VERSION ARG RELEASE_VERSION ARG HADOOP_VERSION ARG HBASE_VERSION +ARG HBASE_THIRDPARTY_VERSION +ARG PROTOBUF_VERSION +ARG PROTOBUF_PLUGIN_VERSION ARG STACKABLE_USER_UID # Patch the hbase-connectors source code @@ -107,12 +110,18 @@ mvn \ --define scala.binary.version="${SCALA_BINARY_VERSION}" \ --define hadoop-three.version="${HADOOP_VERSION}" \ --define hbase.version="${HBASE_VERSION}" \ + --define hbase-thirdparty.version="${HBASE_THIRDPARTY_VERSION}" \ + --define external.protobuf.version="${PROTOBUF_VERSION}" \ + --define protobuf.plugin.version="${PROTOBUF_PLUGIN_VERSION}" \ --define skipTests \ --define maven.test.skip=true \ clean package cp "$(pwd)/hbase-spark/target/hbase-spark-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-${NEW_VERSION}.jar +# The hbase-spark JAR is a plain module JAR and does not bundle its +# dependencies. The generated protobuf classes live here. +cp "$(pwd)/hbase-spark-protocol-shaded/target/hbase-spark-protocol-shaded-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-protocol-shaded-${NEW_VERSION}.jar chmod g=u /stackable/hbase-connector-${NEW_VERSION}-src.tar.gz . EOF diff --git a/spark-k8s/hbase-connectors/boil-config.toml b/spark-k8s/hbase-connectors/boil-config.toml index 54d883165..c0c1301ce 100644 --- a/spark-k8s/hbase-connectors/boil-config.toml +++ b/spark-k8s/hbase-connectors/boil-config.toml @@ -5,4 +5,12 @@ java-devel = "17" spark-version = "3.5.8" # TODO: Bump to 3.4.3. Note that this causes compilation errors that need to be fixed. hadoop-version = "3.4.2" -hbase-version = "2.6.3" +hbase-version = "2.6.6" +# The following three must match the hbase-thirdparty (and its bundled protobuf) +# shipped by the hbase-shaded-client in the final Spark image. That client comes +# from the `hbase` local-image in spark-k8s/boil-config.toml (currently 2.6.6), +# whose hbase-thirdparty is 4.1.13 -> protobuf 4.34.0. Keep these in sync if the +# Spark image's HBASE_VERSION changes. +hbase-thirdparty-version = "4.1.13" +protobuf-version = "4.34.0" +protobuf-plugin-version = "0.6.1"