Preload dependencies in build docker images to reduce downloading at build/test/ci runtime

Additional fixes
 - upgrade ubuntu-test.docker to ubuntu 22.04 (and some small fixes brought forward from CASSANDRA-20997)
 - (.jenkins/k8s) changes gke's small node pool from n2-highcpu-8 to e2-highcpu-8 (gke's own pod resources reqs increased)
 - (.jenkins/k8s) limit one agent pod per node (when jenkins is deployed in k8s using our helm)
 - (.jenkins/k8s) add the cassandra-6.0 job
 - pass maven.repo.local sys prop, if set, into accord's gradle build
 - check and fail on maven dependencies checksums when downloading
 - when on-the-fly building docker images add `--load` (podman compatbility)
 - newer python versions can use venv instead of virtualenv
 - rename ubuntu2004_test.docker to ubuntu-test.docker (in 5.0)
 - (Jenkinsfile) fix retries (which was no longer working when agents/nodes died, since CASSANDRA-20833)
 - (Jenkinsfile) @NonCPS where (now) needed
 - (Jenkinsfile) fix default value for the branch parameter (after a new helm deploy)
 - (.build/run-ci) fix missing parameters on jobs also at helm deploy time
 - (.build/run-ci) .git suffixes on repo urls are optional

 patch by Mick Semb Wever; reviewed by Dmitry Konstantinov, Stefan Miklosovic for CASSANDRA-21403

Author: michaelsembwever

.build/build-resolver.xml

@@ -56,8 +56,8 @@
 
         <typedef uri="antlib:org.apache.maven.resolver.ant" resource="org/apache/maven/resolver/ant/antlib.xml" classpathref="resolver-ant-tasks.classpath" />
         <resolver:remoterepos id="all">
-            <remoterepo id="resolver-central" url="${artifact.remoteRepository.central}"/>
-            <remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}"/>
+            <remoterepo id="resolver-central" url="${artifact.remoteRepository.central}" checksums="fail" />
+            <remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}" checksums="fail" />
             <!-- Snapshot artifacts must not exist in nor be downloaded by any Cassandra release artifact.
                  Please validate that all artifacts included in parent-pom-template.xml are release
                  artifacts before committing.

.build/docker/_create_user.sh

@@ -51,6 +51,12 @@ echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username}
 chmod 0440 /etc/sudoers.d/${username}
 mkdir -p ${BUILD_HOME}/docker ${DIST_DIR} ${BUILD_HOME}/.ssh
 
+# rsync in cached maven dependencies
+echo "Syncing maven dependencies and gradle wrapper"
+rsync -a /home/image-cache/.m2/repository/ ${BUILD_HOME}/.m2/repository/
+cp -a /home/image-cache/.gradle ${BUILD_HOME}/
+chown -R ${username}:${username} ${BUILD_HOME}/.gradle ${BUILD_HOME}/.m2
+
 # we need to make SSH less strict to prevent various dtests from failing when they attempt to
 # git clone a given commit/tag/etc
 echo 'Host *\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no' > ${BUILD_HOME}/.ssh/config

.build/docker/_docker_run.sh

@@ -31,6 +31,8 @@
 # variables, with defaults
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)"
 [ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
+# parameterise the maven repository host directory, as it cannot be shared across containers
+# m2_dir fails under /tmp on macos
 [ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository"
 [ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
 [ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; }
@@ -99,7 +101,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
   if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
     # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
     echo "Building docker image..."
-    until docker build -t ${image_name} -f docker/${dockerfile} .  ; do
+    until docker build -t ${image_name} -f docker/${dockerfile} --load .  ; do
         echo "docker build failed… trying again in 10s… "
         sleep 10
     done

.build/docker/_prepopulate_maven_deps.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Script to prepopulate Maven repository with dependencies from multiple Cassandra branches
+# This will download all dependencies to a custom Maven repository directory
+
+# pre-conditions
+command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; }
+command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; }
+
+
+error() {
+  echo >&2 $2;
+  set -x
+  exit $1
+}
+
+# Function to download dependencies for a branch
+download_deps_for_branch() {
+    local branch=$1
+    local branch_name=$(echo "$branch" | sed 's|origin/||')
+    
+    # Check if branch exists
+    if ! git rev-parse --verify "$branch" >/dev/null 2>&1; then
+        echo "WARNING: Branch $branch does not exist, skipping..."
+        return
+    fi
+    
+    git checkout "$branch"
+    
+    echo ""
+    echo "Downloading dependencies for $branch to $CUSTOM_M2_REPO..."
+    echo ""
+
+    # ensure git modules are initialised
+    ant init
+    # download all dependencies
+    ant -Dmaven.repo.local="$CUSTOM_M2_REPO" -Dlocal.repository="$CUSTOM_M2_REPO" resolver-dist-lib
+}
+
+CUSTOM_M2_REPO="${1:-$HOME/.m2/repository}"
+TMP_DIR=${TMP_DIR:-/tmp}
+
+cd $TMP_DIR
+git clone https://github.com/apache/cassandra.git
+cd cassandra
+git config advice.detachedHead false
+
+# Automatically detect branches from cassandra-5.0 onwards to trunk
+echo "Detecting branches..."
+BRANCHES=()
+
+# Get all origin branches matching cassandra-5.x+, cassandra-6.x+, etc., and trunk
+# Pattern matches: cassandra-5.0, cassandra-5.0.0, cassandra-10.0, cassandra-10.0.1, trunk
+while IFS= read -r branch; do
+    BRANCHES+=("$branch")
+done < <(git branch -r | grep -E "^\s*origin/(cassandra-[5-9][0-9]*\.[0-9]+(\.[0-9]+)?|trunk)$" | sed 's/^[[:space:]]*//' | sort -V)
+
+# If no branches found, fail
+if [ ${#BRANCHES[@]} -eq 0 ]; then
+    echo "ERROR: No branches auto-detected matching pattern origin/cassandra-[5+].x or origin/trunk"
+    echo "Please ensure you have fetched remote branches: git fetch origin"
+    exit 1
+fi
+
+echo "Branches to process:"
+for branch in "${BRANCHES[@]}"; do
+    echo "  - $branch"
+done
+echo "=========================================="
+echo ""
+
+# Create custom Maven repository directory
+mkdir -p "$CUSTOM_M2_REPO"
+
+# Process each branch
+for branch in "${BRANCHES[@]}"; do
+    download_deps_for_branch "$branch"
+done
+
+cd -
+rm -rf $TMP_DIR/cassandra

.build/docker/almalinux-build.docker

@@ -26,7 +26,7 @@ ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
 ARG UID_ARG=1000
 ARG GID_ARG=1000
 
-LABEL org.cassandra.buildenv=almalinux
+LABEL org.cassandra.buildenv=almalinux_build
 
 RUN echo "Building with arguments:" \
     && echo " - DIST_DIR=${DIST_DIR}" \
@@ -55,3 +55,12 @@ RUN rpm -i --nodeps ant-junit-1.9.4-2.el7.noarch.rpm
 
 # python3 is needed for the gen-doc target
 RUN pip3 install --upgrade pip
+
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN alternatives --set java $(alternatives --display java | grep "family java-11-openjdk" | cut -d' ' -f1)
+RUN alternatives --set javac $(alternatives --display javac | grep "family java-11-openjdk" | cut -d' ' -f1)
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::") \
+    bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository && rm /tmp/_prepopulate_maven_deps.sh
+RUN cp -a /root/.gradle /home/image-cache/.gradle

.build/docker/bullseye-build.docker

@@ -23,7 +23,7 @@ ENV DIST_DIR=/dist
 ENV BUILD_HOME=/home/build
 ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
 
-LABEL org.cassandra.buildenv=bullseye
+LABEL org.cassandra.buildenv=debian_build
 
 RUN echo "Building with arguments:" \
     && echo " - DIST_DIR=${DIST_DIR}" \
@@ -70,4 +70,10 @@ RUN sh -c '\
 
 ENV GOROOT="/usr/local/go"
 ENV GOPATH="$BUILD_HOME/go"
-ENV PATH="$PATH:/usr/local/go/bin"
+ENV PATH="$PATH:/usr/local/go/bin"
+
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository && rm /tmp/_prepopulate_maven_deps.sh
+RUN cp -a /root/.gradle /home/image-cache/.gradle

.build/docker/run-tests.sh

@@ -25,6 +25,8 @@
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)"
 [ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
 [ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
+# parameterise the maven repository host directory, as it cannot be shared across containers.
+# m2_dir fails under /tmp on macos
 [ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository"
 [ "x${docker_timeout_hours}" != "x" ] || docker_timeout_hours="1"
 [ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
@@ -134,7 +136,7 @@ docker --version
 pushd ${cassandra_dir}/.build >/dev/null
 
 # build test image
-dockerfile="ubuntu2004_test.docker"
+dockerfile="ubuntu-test.docker"
 image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
 image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
 docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${m2_dir}:/home/cassandra/.m2/repository"
@@ -147,7 +149,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
   if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
     # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
     echo "Building docker image..."
-    until docker build -t ${image_name} -f docker/${dockerfile} .  ; do
+    until docker build -t ${image_name} -f docker/${dockerfile} --load .  ; do
       echo "docker build failed… trying again in 10s… "
       sleep 10
     done
@@ -294,7 +296,7 @@ docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_versi
 # start the container, timeout after 4 hours
 docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep ${docker_timeout_hours}h)
 
-echo "Running container ${container_name} ${docker_id}"
+echo "Running container ${container_name} ${docker_id} using image ${image_name}"
 
 docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh cassandra $(id -u) $(id -g)" | tee -a ${logfile}
 docker exec --user root ${container_name} update-alternatives --set python /usr/bin/python${python_version} | tee -a ${logfile}

.build/docker/ubuntu2004_test.docker .build/docker/ubuntu-test.docker.build/docker/ubuntu2004_test.docker renamed to .build/docker/ubuntu-test.docker

@@ -10,8 +10,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:20.04
-MAINTAINER Apache Cassandra <dev@cassandra.apache.org>
+FROM ubuntu:22.04
+LABEL org.opencontainers.image.authors="Apache Cassandra <dev@cassandra.apache.org>"
 
 # CONTEXT is expected to be cassandra/.build
 
@@ -23,7 +23,7 @@ ENV LC_CTYPE=en_US.UTF-8
 ENV PYTHONIOENCODING=utf-8
 ENV PYTHONUNBUFFERED=true
 
-LABEL org.cassandra.buildenv=ubuntu_2004
+LABEL org.cassandra.buildenv=ubuntu_test
 
 RUN echo "Building with arguments:" \
     && echo " - DIST_DIR=${DIST_DIR}" \
@@ -42,7 +42,7 @@ RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y --no-install-recommends software-properties-common apt-utils
+    apt-get install -y --no-install-recommends software-properties-common apt-utils gnupg
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
@@ -54,7 +54,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         vim lsof sudo libjemalloc2 dumb-init locales rsync \
         openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
 
-
+RUN update-alternatives --remove java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/jre/bin/java
+RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java 1081
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 3
 RUN python3.8 -m pip install --upgrade pip
@@ -79,6 +80,7 @@ RUN find /etc -type f -name java.security -exec sed -i 's/TLSv1, TLSv1.1//' {} \
 RUN find /etc -type f -name java.security -exec sed -i 's/3DES_EDE_CBC$/3DES_EDE_CBC, TLSv1, TLSv1.1/' {} \;
 
 # create and change to cassandra-tmp user, use an rare uid to avoid collision later on
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
 RUN adduser --disabled-login --uid 901743 --lastuid 901743 --gecos cassandra cassandra-tmp
 RUN gpasswd -a cassandra-tmp sudo
 RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
@@ -87,11 +89,16 @@ RUN chmod 0440 /etc/sudoers.d/build
 # switch to the cassandra user
 RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
 USER cassandra-tmp
-ENV HOME ${BUILD_HOME}
+ENV HOME=${BUILD_HOME}
 WORKDIR ${BUILD_HOME}
 
 ENV ANT_HOME=/usr/share/ant
 
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository
+RUN cp -a /home/cassandra-tmp/.gradle /home/image-cache/.gradle
+
 # run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we
 # setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.8/3.11
 # Don't build cython extensions when installing cassandra-driver. During test execution the driver
@@ -110,15 +117,22 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
     && pip3 install -r /opt/requirements.txt \
     && pip3 freeze --user"
 
-RUN virtualenv --python=python3.11 ${BUILD_HOME}/env3.11
+RUN python3.11 -m venv ${BUILD_HOME}/env3.11
 RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate
 
 RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
     && source ${BUILD_HOME}/env3.11/bin/activate \
     && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \
-    && pip3 install -r /opt/requirements.txt \
+    && pip3 install --upgrade \"pip<25.0\"  \"setuptools==60.8.2\" wheel \
+    && pip3 install --no-build-isolation -r /opt/requirements.txt \
     && pip3 freeze --user"
 
+# 4* requires java8, sudo doesn't work on cross-platform builds
+USER root
+RUN update-alternatives --set java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/javac
+USER cassandra-tmp
+
 # Initialize the CCM git repo as well as this also can fail to clone
 RUN /bin/bash -c "source ${BUILD_HOME}/env3.8/bin/activate && \
     ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
@@ -131,16 +145,23 @@ RUN bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
     latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
     for i in $(seq 1 $latest_4_1); do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
 
-# 5+ requires java11
-RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture)
+# 5+ requires java11, sudo doesn't work on cross-platform builds
+USER root
+RUN update-alternatives --set java /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/javac
+USER cassandra-tmp
 
 # Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
 RUN rm -fr ${BUILD_HOME}/.ccm/repository/_git_cache_apache
 RUN /bin/bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
     ccm create --quiet -n 1 -v git:cassandra-5.0 test && ccm remove test && \
+    ccm create --quiet -n 1 -v git:cassandra-6.0 test && ccm remove test && \
     ccm create --quiet -n 1 -v git:trunk test && ccm remove test && \
     latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
     for i in $(seq 1 $latest_5_0); do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done'
+    # TODO uncomment when 6.0.0 is released
+    #latest_6_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")6\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
+    #for i in $(seq 1 $latest_6_0); do echo $i ; ccm create --quiet -n 1 -v binary:6.0.$i test && ccm remove test ; done'
 
 # the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them
 #  and other directories we don't need in image