Skip to content

Commit af0a4eb

Browse files
sbernauersbernauer
committed
WIP: Move s3a and abfs jar collection into hadoop builder image
1 parent cde4b7c commit af0a4eb

8 files changed

Lines changed: 64 additions & 101 deletions

File tree

‎hadoop/hadoop/Dockerfile‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ FROM local-image/java-devel AS hadoop-builder
77
ARG PRODUCT_VERSION
88
ARG RELEASE_VERSION
99
ARG PROTOBUF_VERSION
10+
ARG AWS_JAVA_SDK_BUNDLE_VERSION
11+
ARG AZURE_STORAGE_VERSION
12+
ARG AZURE_KEYVAULT_CORE_VERSION
1013
ARG STACKABLE_USER_UID
1114

1215
WORKDIR /stackable
@@ -90,6 +93,7 @@ tar -czf /stackable/hadoop-${NEW_VERSION}-src.tar.gz .
9093
# Therefore, this build does work but the final image does NOT contain the openssl-devel package which is why it fails there which is why we have to create the symlink over there manually.
9194
# We still leave this flag in to automatically fail should anything with the packages or symlinks ever fail.
9295
mvn \
96+
--threads 1C \
9397
clean package install \
9498
-Pdist,native \
9599
-pl '!hadoop-tools/hadoop-pipes' \
@@ -140,3 +144,28 @@ rm -rf /stackable/.m2
140144
# Set correct groups; make sure only required artifacts for the final image are located in /stackable
141145
chmod -R g=u /stackable
142146
EOF
147+
148+
RUN <<EOF
149+
NEW_VERSION=${PRODUCT_VERSION}-stackable${RELEASE_VERSION}
150+
151+
# Place cloud libraries (for s3a:// and abfs://) in the /stackable/hadoop-cloud-libraries/ folder to
152+
# have one central place for dependant images to pull them from
153+
mkdir -p /stackable/hadoop-cloud-libraries/
154+
155+
if [[ "$PRODUCT_VERSION" == "3.3.6" || "$PRODUCT_VERSION" == "3.4.2" ]]; then
156+
cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
157+
cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
158+
cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
159+
cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
160+
cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
161+
else
162+
cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
163+
# Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more.
164+
# TODO: Use Nexus, but I currently can't upload.
165+
# curl -o /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
166+
curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_JAVA_SDK_BUNDLE_VERSION}/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
167+
cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/
168+
cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/
169+
cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/
170+
fi
171+
EOF

‎hadoop/hadoop/boil-config.toml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ java-devel = "11"
44

55
[versions."3.3.6".build-arguments]
66
protobuf-version = "3.7.1" # https://github.com/apache/hadoop/blob/rel/release-3.3.6/BUILDING.txt
7+
aws-java-sdk-bundle-version = "1.12.367" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
8+
azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6
9+
azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
710

811

912
[versions."3.4.2".local-images]
@@ -12,6 +15,10 @@ java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Ja
1215

1316
[versions."3.4.2".build-arguments]
1417
protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt
18+
aws-java-sdk-bundle-version = "2.29.52" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.2
19+
azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.2
20+
azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
21+
1522

1623

1724
[versions."3.4.3".local-images]
@@ -20,6 +27,10 @@ java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Ja
2027

2128
[versions."3.4.3".build-arguments]
2229
protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt
30+
aws-java-sdk-bundle-version = "2.35.4" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
31+
azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
32+
azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
33+
2334

2435

2536
[versions."3.5.0".local-images]
@@ -28,3 +39,6 @@ java-devel = "17" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Ja
2839

2940
[versions."3.5.0".build-arguments]
3041
protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt
42+
aws-java-sdk-bundle-version = "2.35.4" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.5.0
43+
azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.5.0
44+
azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1

‎hbase/Dockerfile‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,8 @@ ARG STACKABLE_USER_UID
3030
USER ${STACKABLE_USER_UID}
3131
WORKDIR /stackable
3232

33-
COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
34-
# The artifact name of the AWS bundle has changed between Haddop 3.3.6 and 3.4.1
35-
# from aws-java-sdk-bundle-*.jar to bundle-*.jar.
36-
# See: https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
37-
# So we try to copy both and if one of them doesn't exist buildx will just ignore it :)
38-
/stackable/hadoop/share/hadoop/tools/lib/bundle-*.jar \
39-
/stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-*.jar \
40-
/stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \
41-
/stackable/hadoop/share/hadoop/tools/lib/
33+
# Copy cloud libraries for s3a:// and abfs:// support
34+
COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-cloud-libraries/*.jar /stackable/hadoop/share/hadoop/tools/lib/
4235

4336
COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase/stackable/bin/export-snapshot-to-s3.env /stackable/bin/
4437

‎hive/Dockerfile‎

Lines changed: 15 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ ARG HADOOP_HADOOP_VERSION
1818
# Reassign the arg to `HADOOP_VERSION` for better readability.
1919
ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION}
2020
ARG JMX_EXPORTER_VERSION
21-
ARG AWS_JAVA_SDK_BUNDLE_VERSION
22-
ARG AZURE_STORAGE_VERSION
23-
ARG AZURE_KEYVAULT_CORE_VERSION
2421
ARG STACKABLE_USER_UID
2522

2623
# Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
@@ -64,18 +61,20 @@ tar -czf /stackable/hive-${NEW_VERSION}-src.tar.gz .
6461

6562
if [[ "${PRODUCT_VERSION}" == "3.1.3" ]] ; then
6663
mvn \
67-
clean package \
68-
-DskipTests \
64+
--threads 1C \
65+
clean package \
66+
-DskipTests \
6967
--projects standalone-metastore
7068
mv standalone-metastore/target/apache-hive-metastore-${NEW_VERSION}-bin/apache-hive-metastore-${NEW_VERSION}-bin /stackable
7169
mv standalone-metastore/target/bom.json /stackable/apache-hive-metastore-${NEW_VERSION}-bin/apache-hive-metastore-${NEW_VERSION}.cdx.json
7270
elif [[ "${PRODUCT_VERSION}" == 4.0.* ]]; then
7371
(
7472
# https://issues.apache.org/jira/browse/HIVE-20451 switched the metastore server packaging starting with 4.0.0
7573
mvn \
76-
clean package \
77-
-DskipTests \
78-
-Dhadoop.version=${HADOOP_VERSION}-stackable${RELEASE_VERSION}
74+
--threads 1C \
75+
clean package \
76+
-DskipTests \
77+
-Dhadoop.version=${HADOOP_VERSION}-stackable${RELEASE_VERSION}
7978

8079
# We only seem to get a .tar.gz archive, so let's extract that to the correct location
8180
tar --extract --directory=/stackable -f standalone-metastore/metastore-server/target/apache-hive-standalone-metastore-server-${NEW_VERSION}-bin.tar.gz
@@ -86,11 +85,11 @@ elif [[ "${PRODUCT_VERSION}" == 4.0.* ]]; then
8685
)
8786
else
8887
# Starting with 4.1.0 the build process changed again in https://github.com/apache/hive/pull/5936 (HIVE-29062)
89-
mvn \
90-
clean package \
91-
-Dhadoop.version=${HADOOP_VERSION}-stackable${RELEASE_VERSION} \
92-
-DskipTests \
93-
-Pdist
88+
mvn --threads 1C \
89+
clean package \
90+
-Dhadoop.version=${HADOOP_VERSION}-stackable${RELEASE_VERSION} \
91+
-DskipTests \
92+
-Pdist
9493
# Looks like we can not filter the projects using "--projects standalone-metastore/metastore-server --also-make",
9594
# as this does not build a *.tar.gz
9695

@@ -103,6 +102,9 @@ fi
103102
rm -rf "$BUILD_SRC_DIR"
104103
EOF
105104

105+
# Copy cloud libraries for s3a:// and abfs:// support
106+
COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-cloud-libraries/*.jar /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/lib/
107+
106108
RUN <<EOF
107109
cd /stackable
108110
mkdir /stackable/jmx
@@ -112,25 +114,6 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" /sta
112114
# Needed to run housekeeping jobs, see footnote <1> below
113115
cp /stackable/patched-libs/maven/org/apache/hadoop/hadoop-mapreduce-client-core/${HADOOP_VERSION}-stackable${RELEASE_VERSION}/hadoop-mapreduce-client-core-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/lib/
114116

115-
# The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards
116-
# This way the build will fail should one of the files not be available anymore in a later Hadoop version!
117-
118-
# Add S3 Support for Hive (support for s3a://)
119-
cp /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/lib/
120-
121-
# According to https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html, the jar filename has changed from
122-
# aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar to bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar. In future, you might need to do:
123-
if [[ "${PRODUCT_VERSION}" == "3.1.3" || "${PRODUCT_VERSION}" == 4.0.* ]]; then
124-
cp /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/lib/
125-
else
126-
cp /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/lib/
127-
fi
128-
129-
# Add Azure ABFS support (support for abfs://)
130-
cp /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/
131-
cp /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/
132-
cp /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/
133-
134117
# We're removing these to make the intermediate layer smaller
135118
# This can be necessary even though it's only a builder image because the GitHub Action Runners only have very limited space available
136119
# and we are sometimes running into errors because we're out of space.

‎hive/boil-config.toml‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,6 @@ java-devel = "8"
1111

1212
[versions."3.1.3".build-arguments]
1313
jmx-exporter-version = "1.3.0"
14-
# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
15-
aws-java-sdk-bundle-version = "1.12.367"
16-
azure-storage-version = "7.0.1"
17-
azure-keyvault-core-version = "1.0.0"
1814

1915
[versions."4.0.0".local-images]
2016
# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11
@@ -26,10 +22,6 @@ java-devel = "8"
2622

2723
[versions."4.0.0".build-arguments]
2824
jmx-exporter-version = "1.3.0"
29-
# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
30-
aws-java-sdk-bundle-version = "1.12.367"
31-
azure-storage-version = "7.0.1"
32-
azure-keyvault-core-version = "1.0.0"
3325

3426
[versions."4.0.1".local-images]
3527
# Hive 4.0 must be built with Java 8 (according to GitHub README) but seems to run on Java 11
@@ -41,10 +33,6 @@ java-devel = "8"
4133

4234
[versions."4.0.1".build-arguments]
4335
jmx-exporter-version = "1.3.0"
44-
# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
45-
aws-java-sdk-bundle-version = "1.12.367"
46-
azure-storage-version = "7.0.1"
47-
azure-keyvault-core-version = "1.0.0"
4836

4937
[versions."4.1.0".local-images]
5038
# Hive 4.1 requires Java 17 (according to GitHub README)
@@ -56,9 +44,6 @@ java-devel = "17"
5644

5745
[versions."4.1.0".build-arguments]
5846
jmx-exporter-version = "1.3.0"
59-
aws-java-sdk-bundle-version = "2.35.4" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
60-
azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
61-
azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
6247

6348
[versions."4.2.0".local-images]
6449
# Hive 4.2 requires Java 21 (according to GitHub README)
@@ -70,6 +55,3 @@ java-devel = "21"
7055

7156
[versions."4.2.0".build-arguments]
7257
jmx-exporter-version = "1.3.0"
73-
aws-java-sdk-bundle-version = "2.35.4" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3
74-
azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3
75-
azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1

‎spark-k8s/Dockerfile.3‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@ ARG HADOOP_HADOOP_VERSION
4444
# Reassign the arg to `HADOOP_VERSION` for better readability.
4545
ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION}
4646
ARG HBASE_VERSION
47-
ARG AWS_JAVA_SDK_BUNDLE_VERSION
48-
ARG AZURE_STORAGE_VERSION
49-
ARG AZURE_KEYVAULT_CORE_VERSION
5047
ARG JACKSON_DATAFORMAT_XML_VERSION
5148
ARG STAX2_API_VERSION
5249
ARG WOODSTOX_CORE_VERSION
@@ -108,18 +105,8 @@ EOF
108105

109106
WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/jars
110107

111-
# Copy modules required for s3a://
112-
COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
113-
/stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \
114-
/stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar \
115-
./
116-
117-
# Copy modules required for abfs://
118-
COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
119-
/stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \
120-
/stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar \
121-
/stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar \
122-
./
108+
# Copy cloud libraries for s3a:// and abfs:// support
109+
COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-cloud-libraries/*.jar ./
123110

124111
# Copy the HBase connector including required modules
125112
COPY --from=hbase-connectors-builder --chown=${STACKABLE_USER_UID}:0 \

‎spark-k8s/Dockerfile.4‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,6 @@ ARG PRODUCT_VERSION
3636
ARG HADOOP_HADOOP_VERSION
3737
# Reassign the arg to `HADOOP_VERSION` for better readability.
3838
ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION}
39-
ARG AWS_JAVA_SDK_BUNDLE_VERSION
40-
ARG AZURE_STORAGE_VERSION
41-
ARG AZURE_KEYVAULT_CORE_VERSION
4239
ARG JACKSON_DATAFORMAT_XML_VERSION
4340
ARG STAX2_API_VERSION
4441
ARG WOODSTOX_CORE_VERSION
@@ -104,18 +101,8 @@ EOF
104101

105102
WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/jars
106103

107-
# Copy modules required for s3a://
108-
COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
109-
/stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \
110-
/stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar \
111-
./
112-
113-
# Copy modules required for abfs://
114-
COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
115-
/stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \
116-
/stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar \
117-
/stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar \
118-
./
104+
# Copy cloud libraries for s3a:// and abfs:// support
105+
COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-cloud-libraries/*.jar ./
119106

120107
COPY spark-k8s/stackable/jmx /stackable/jmx
121108

0 commit comments

Comments
 (0)