Skip to content

Commit f8cc523

Browse files
kevinjqliuCopilot
andcommitted
try 1.11.0 rc1
Co-authored-by: Copilot <copilot@github.com>
1 parent 2dadcdf commit f8cc523

1 file changed

Lines changed: 13 additions & 2 deletions

File tree

dev/spark/Dockerfile

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@ FROM apache/spark:${BASE_IMAGE_SPARK_VERSION}
1919

2020
# Dependency versions - keep these compatible
2121
# Changing these will invalidate the JAR download cache layer
22-
ARG ICEBERG_VERSION=1.10.1
22+
ARG ICEBERG_VERSION=1.11
2323
ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13
2424
ARG HADOOP_VERSION=3.4.1
2525
ARG AWS_SDK_VERSION=2.24.6
2626
ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2
27+
# ARG ICEBERG_MAVEN_MIRROR=${MAVEN_MIRROR}
28+
ARG ICEBERG_MAVEN_MIRROR=https://repository.apache.org/content/repositories/orgapacheiceberg-1278
2729

2830
USER root
2931
WORKDIR ${SPARK_HOME}
@@ -38,11 +40,20 @@ RUN apt-get update -qq && \
3840

3941
# Download JARs with retry logic (most cacheable - only changes when versions change)
4042
# This is the slowest step, so we do it before copying config files
43+
# Iceberg JARs use ICEBERG_MAVEN_MIRROR (defaults to MAVEN_MIRROR, can be overridden for staging repos)
4144
RUN set -e && \
4245
cd "${SPARK_HOME}/jars" && \
4346
for jar_path in \
4447
"org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" \
45-
"org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" \
48+
"org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar"; \
49+
do \
50+
jar_name=$(basename "${jar_path}") && \
51+
curl -fsSL --retry 3 --retry-delay 5 \
52+
-o "${jar_name}" \
53+
"${ICEBERG_MAVEN_MIRROR}/${jar_path}" && \
54+
chown spark:spark "${jar_name}"; \
55+
done && \
56+
for jar_path in \
4657
"org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" \
4758
"software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"; \
4859
do \

0 commit comments

Comments
 (0)