Bump maven, Java, and PySpark to support Spark connect (#80)

Author: trivialfis

containers/conda_env/aarch64_test.yml

@@ -27,7 +27,12 @@ dependencies:
 - llvmlite
 - loky>=3.5.1
 - pyarrow
-- pyspark>=3.4.0,<4.0
+# PySpark
+- pyspark>=4.0
+- grpcio
+- grpcio-status
+- googleapis-common-protos
+- zstandard
 - cloudpickle
 - pip:
   - awscli

containers/conda_env/linux_cpu_test.yml

@@ -35,6 +35,11 @@ dependencies:
 - protobuf
 - cloudpickle
 - modin
-- pyspark>=3.4.0,<4.0
+# PySpark
+- pyspark>=4.0
+- grpcio
+- grpcio-status
+- googleapis-common-protos
+- zstandard
 - pip:
   - py-ubjson

containers/dockerfile/Dockerfile.cpu

@@ -10,7 +10,7 @@ ENV CC=gcc-10
 ENV CXX=g++-10
 ENV CPP=cpp-10
 ENV GOSU_VERSION=1.10
-ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/
 
 # Install all basic requirements
 RUN \
@@ -19,7 +19,7 @@ RUN \
     apt-get install -y software-properties-common && \
     add-apt-repository ppa:ubuntu-toolchain-r/test && \
     apt-get update && \
-    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \
+    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-17-jdk-headless && \
     # Miniforge
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-${ARCH}.sh && \
     bash conda.sh -b -p /opt/miniforge

containers/dockerfile/Dockerfile.gpu

@@ -13,7 +13,7 @@ SHELL ["/bin/bash", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PATH=/opt/miniforge/bin:$PATH
 ENV GOSU_VERSION=1.10
-ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/
 
 # Install all basic requirements
 RUN \
@@ -23,7 +23,7 @@ RUN \
     sed -i 's/ports.ubuntu.com/mirrors.ocf.berkeley.edu/g' /etc/apt/sources.list && \
     apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${CUDA_REPO_ARCH}/3bf863cc.pub && \
     apt-get update && \
-    apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \
+    apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-17-jdk-headless && \
     apt-get install "libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT}" \
       "libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT}" -y --allow-change-held-packages && \
     # Miniforge
@@ -43,8 +43,8 @@ RUN \
         distributed \
         "dask-cuda=$RAPIDS_VERSION.*" "dask-cudf=$RAPIDS_VERSION.*" cupy \
         numpy pytest pytest-timeout scipy scikit-learn pandas matplotlib wheel \
-        python-kubernetes urllib3 graphviz hypothesis "loky>=3.5.1" \
-        "pyspark>=3.4.0,<4.0" cloudpickle cuda-python && \
+        python-kubernetes urllib3 graphviz hypothesis "loky>=3.5.1" cuda-python \
+        "pyspark>=4.0" grpcio grpcio-status googleapis-common-protos zstandard cloudpickle && \
     mamba clean --all --yes
 
 # Install lightweight sudo (not bound to TTY)

containers/dockerfile/Dockerfile.i386

@@ -5,8 +5,8 @@ SHELL ["/bin/bash", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive
 ENV GOSU_VERSION=1.10
 
-RUN apt-get update && \
-    apt-get install -y tar unzip wget git build-essential ninja-build cmake curl
+RUN apt-get update && apt-get upgrade -y  && \
+    apt-get install -y tar unzip wget git build-essential ninja-build cmake curl ca-certificates
 
 # Install lightweight sudo (not bound to TTY)
 RUN set -ex; \

containers/dockerfile/Dockerfile.jvm

@@ -1,7 +1,7 @@
 FROM rockylinux:8
 ARG MINIFORGE_VERSION=24.9.2-0
 ARG CMAKE_VERSION=3.31.2
-ARG MAVEN_VERSION=3.9.12
+ARG MAVEN_VERSION=3.9.14
 
 SHELL ["/bin/bash", "-c"]
 
@@ -15,7 +15,7 @@ ENV GOSU_VERSION=1.10
 RUN dnf -y update && \
     dnf -y install dnf-plugins-core && \
     dnf config-manager --set-enabled powertools && \
-    dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \
+    dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-17-openjdk-devel \
                    gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \
                    gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \
     # Miniforge

containers/dockerfile/Dockerfile.jvm_gpu_build

@@ -4,7 +4,7 @@ ARG CUDA_VERSION
 ARG NCCL_VERSION
 ARG MINIFORGE_VERSION=24.9.2-0
 ARG CMAKE_VERSION=3.31.2
-ARG MAVEN_VERSION=3.9.12
+ARG MAVEN_VERSION=3.9.14
 
 SHELL ["/bin/bash", "-c"]
 
@@ -21,7 +21,7 @@ RUN \
     dnf -y update && \
     dnf -y install dnf-plugins-core && \
     dnf config-manager --set-enabled powertools && \
-    dnf install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
+    dnf install -y tar unzip wget xz git which ninja-build java-17-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \
     # Miniforge
     wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \
     bash conda.sh -b -p /opt/miniforge && \