diff --git a/containers/conda_env/aarch64_test.yml b/containers/conda_env/aarch64_test.yml index bb9a186..8944de0 100644 --- a/containers/conda_env/aarch64_test.yml +++ b/containers/conda_env/aarch64_test.yml @@ -27,7 +27,12 @@ dependencies: - llvmlite - loky>=3.5.1 - pyarrow -- pyspark>=3.4.0,<4.0 +# PySpark +- pyspark>=4.0 +- grpcio +- grpcio-status +- googleapis-common-protos +- zstandard - cloudpickle - pip: - awscli diff --git a/containers/conda_env/linux_cpu_test.yml b/containers/conda_env/linux_cpu_test.yml index 846bb35..c5e8023 100644 --- a/containers/conda_env/linux_cpu_test.yml +++ b/containers/conda_env/linux_cpu_test.yml @@ -35,6 +35,11 @@ dependencies: - protobuf - cloudpickle - modin -- pyspark>=3.4.0,<4.0 +# PySpark +- pyspark>=4.0 +- grpcio +- grpcio-status +- googleapis-common-protos +- zstandard - pip: - py-ubjson diff --git a/containers/dockerfile/Dockerfile.cpu b/containers/dockerfile/Dockerfile.cpu index e40c90b..15fc560 100644 --- a/containers/dockerfile/Dockerfile.cpu +++ b/containers/dockerfile/Dockerfile.cpu @@ -10,7 +10,7 @@ ENV CC=gcc-10 ENV CXX=g++-10 ENV CPP=cpp-10 ENV GOSU_VERSION=1.10 -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/ # Install all basic requirements RUN \ @@ -19,7 +19,7 @@ RUN \ apt-get install -y software-properties-common && \ add-apt-repository ppa:ubuntu-toolchain-r/test && \ apt-get update && \ - apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-8-jdk-headless && \ + apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libidn12 cmake ninja-build gcc-10 g++-10 openjdk-17-jdk-headless && \ # Miniforge wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-${ARCH}.sh && \ bash conda.sh -b -p /opt/miniforge diff --git a/containers/dockerfile/Dockerfile.gpu b/containers/dockerfile/Dockerfile.gpu index 7d8a089..0fb06e6 100644 --- a/containers/dockerfile/Dockerfile.gpu +++ b/containers/dockerfile/Dockerfile.gpu @@ -13,7 +13,7 @@ SHELL ["/bin/bash", "-c"] ENV DEBIAN_FRONTEND=noninteractive ENV PATH=/opt/miniforge/bin:$PATH ENV GOSU_VERSION=1.10 -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/ # Install all basic requirements RUN \ @@ -23,7 +23,7 @@ RUN \ sed -i 's/ports.ubuntu.com/mirrors.ocf.berkeley.edu/g' /etc/apt/sources.list && \ apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${CUDA_REPO_ARCH}/3bf863cc.pub && \ apt-get update && \ - apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \ + apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-17-jdk-headless && \ apt-get install "libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT}" \ "libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT}" -y --allow-change-held-packages && \ # Miniforge @@ -43,8 +43,8 @@ RUN \ distributed \ "dask-cuda=$RAPIDS_VERSION.*" "dask-cudf=$RAPIDS_VERSION.*" cupy \ numpy pytest pytest-timeout scipy scikit-learn pandas matplotlib wheel \ - python-kubernetes urllib3 graphviz hypothesis "loky>=3.5.1" \ - "pyspark>=3.4.0,<4.0" cloudpickle cuda-python && \ + python-kubernetes urllib3 graphviz hypothesis "loky>=3.5.1" cuda-python \ + "pyspark>=4.0" grpcio grpcio-status googleapis-common-protos zstandard cloudpickle && \ mamba clean --all --yes # Install lightweight sudo (not bound to TTY) diff --git a/containers/dockerfile/Dockerfile.i386 b/containers/dockerfile/Dockerfile.i386 index 5ecae14..5643e17 100644 --- a/containers/dockerfile/Dockerfile.i386 +++ b/containers/dockerfile/Dockerfile.i386 @@ -5,8 +5,8 @@ SHELL ["/bin/bash", "-c"] ENV DEBIAN_FRONTEND=noninteractive ENV GOSU_VERSION=1.10 -RUN apt-get update && \ - apt-get install -y tar unzip wget git build-essential ninja-build cmake curl +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y tar unzip wget git build-essential ninja-build cmake curl ca-certificates # Install lightweight sudo (not bound to TTY) RUN set -ex; \ diff --git a/containers/dockerfile/Dockerfile.jvm b/containers/dockerfile/Dockerfile.jvm index aea4f15..c243712 100644 --- a/containers/dockerfile/Dockerfile.jvm +++ b/containers/dockerfile/Dockerfile.jvm @@ -1,7 +1,7 @@ FROM rockylinux:8 ARG MINIFORGE_VERSION=24.9.2-0 ARG CMAKE_VERSION=3.31.2 -ARG MAVEN_VERSION=3.9.12 +ARG MAVEN_VERSION=3.9.14 SHELL ["/bin/bash", "-c"] @@ -15,7 +15,7 @@ ENV GOSU_VERSION=1.10 RUN dnf -y update && \ dnf -y install dnf-plugins-core && \ dnf config-manager --set-enabled powertools && \ - dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-1.8.0-openjdk-devel \ + dnf install -y tar unzip make bzip2 wget xz git which ninja-build java-17-openjdk-devel \ gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ \ gcc-toolset-10-runtime gcc-toolset-10-libstdc++-devel && \ # Miniforge diff --git a/containers/dockerfile/Dockerfile.jvm_gpu_build b/containers/dockerfile/Dockerfile.jvm_gpu_build index c396e1d..c824419 100644 --- a/containers/dockerfile/Dockerfile.jvm_gpu_build +++ b/containers/dockerfile/Dockerfile.jvm_gpu_build @@ -4,7 +4,7 @@ ARG CUDA_VERSION ARG NCCL_VERSION ARG MINIFORGE_VERSION=24.9.2-0 ARG CMAKE_VERSION=3.31.2 -ARG MAVEN_VERSION=3.9.12 +ARG MAVEN_VERSION=3.9.14 SHELL ["/bin/bash", "-c"] @@ -21,7 +21,7 @@ RUN \ dnf -y update && \ dnf -y install dnf-plugins-core && \ dnf config-manager --set-enabled powertools && \ - dnf install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \ + dnf install -y tar unzip wget xz git which ninja-build java-17-openjdk-devel gcc-toolset-10-gcc gcc-toolset-10-binutils gcc-toolset-10-gcc-c++ && \ # Miniforge wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh && \ bash conda.sh -b -p /opt/miniforge && \