Support Python 3.12 and 3.13 in cqlsh

patch by Arvind Kandpal; reviewed by Stefan Miklosovic, Michael Semb Wever for CASSANDRA-20997

Author: arvindKandpal-ksolves

.build/build-resolver.xml

@@ -247,6 +247,7 @@
             <file file="${local.repository}/org/apache/cassandra/deps/geomet-0.1.0.zip"/>
             <file file="${local.repository}/org/apache/cassandra/deps/pure_sasl-0.6.2-py2-none-any.zip"/>
             <file file="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.5-py2.py3-none-any.zip"/>
+            <file file="${local.repository}/org/apache/cassandra/deps/pyasyncore-1.0.5-py3-none-any.zip"/>
         </copy>
         <copy todir="${build.lib}/sigar-bin/" quiet="true">
             <file file="${local.repository}/org/apache/cassandra/deps/sigar-bin/libsigar-amd64-freebsd-6.so"/>
@@ -284,6 +285,7 @@
         <get src="${artifact.python.pypi}/59/a0/cf4cd997e1750f0c2d91c6ea5abea218251c43c3581bcc2f118b00baf5cf/futures-2.1.6-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/futures-2.1.6-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
         <get src="${artifact.python.pypi}/37/b2/ef1124540ee2c0b417be8d0f74667957e6aa084a3f26621aa67e2e77f3fb/pure_sasl-0.6.2-py2-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/pure_sasl-0.6.2-py2-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
         <get src="${artifact.python.pypi}/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/wcwidth-0.2.5-py2.py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
+        <get src="${artifact.python.pypi}/1f/ab/b10cee56269ae150763f3f83b3e9305a11f42f50b3dcd58eeb8f7988f0bb/pyasyncore-1.0.5-py3-none-any.whl" dest="${local.repository}/org/apache/cassandra/deps/pyasyncore-1.0.5-py3-none-any.zip" usetimestamp="true" quiet="true" skipexisting="true"/>
 
         <!-- apache/cassandra/lib -->
         <get src="${lib.download.base.url}/lib/geomet-0.1.0.zip" dest="${local.repository}/org/apache/cassandra/deps/geomet-0.1.0.zip" usetimestamp="true" quiet="true" skipexisting="true"/>

.build/docker/run-tests.sh

@@ -300,6 +300,7 @@ echo "Running container ${container_name} ${docker_id} using image ${image_name}
 
 docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh cassandra $(id -u) $(id -g)" | tee -a ${logfile}
 docker exec --user root ${container_name} update-alternatives --set python /usr/bin/python${python_version} | tee -a ${logfile}
+docker exec --user root ${container_name} update-alternatives --set python3 /usr/bin/python${python_version} | tee -a ${logfile}
 
 if [ -n "${DTEST_TMPDIR_LOCAL}" ] && [[ "${target}" =~ ^dtest-upgrade ]] ; then
     # prepopulate a tmp ccm repository directory, if running dtest-upgrade tests

.build/docker/ubuntu-test.docker

@@ -49,15 +49,26 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y curl git-core python3-pip \
         python3.8 python3.8-venv python3.8-dev \
+        python3.10 python3.10-venv python3.10-dev \
         python3.11 python3.11-venv python3.11-dev \
+        python3.12 python3.12-venv python3.12-dev \
+        python3.13 python3.13-venv python3.13-dev \
         virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \
         vim lsof sudo libjemalloc2 dumb-init locales rsync \
         openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
 
 RUN update-alternatives --remove java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/jre/bin/java
 RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java 1081
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 3
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.13 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 1
 RUN python3.8 -m pip install --upgrade pip
 
 # generate locales for the standard en_US.UTF8 value we use for testing
@@ -100,7 +111,7 @@ RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository
 RUN cp -a /home/cassandra-tmp/.gradle /home/image-cache/.gradle
 
 # run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we
-# setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.8/3.11
+# setup the virtualenv for the cassandra user, not root) for Python 3.8-3.13
 # Don't build cython extensions when installing cassandra-driver. During test execution the driver
 # dependency is refreshed via pip install --upgrade, so that driver changes can be pulled in without
 # requiring the image to be rebuilt. Rebuilding compiled extensions is costly and is disabled by
@@ -117,6 +128,16 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
     && pip3 install -r /opt/requirements.txt \
     && pip3 freeze --user"
 
+RUN virtualenv --python=python3.10 ${BUILD_HOME}/env3.10
+RUN chmod +x ${BUILD_HOME}/env3.10/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+    && source ${BUILD_HOME}/env3.10/bin/activate \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 \
+    && pip3 install --upgrade \"pip<25.0\" \"setuptools==60.8.2\" wheel \
+    && pip3 install --no-build-isolation -r /opt/requirements.txt \
+    && pip3 freeze --user"
+
 RUN python3.11 -m venv ${BUILD_HOME}/env3.11
 RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate
 
@@ -127,6 +148,28 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
     && pip3 install --no-build-isolation -r /opt/requirements.txt \
     && pip3 freeze --user"
 
+RUN virtualenv --python=python3.12 ${BUILD_HOME}/env3.12
+RUN chmod +x ${BUILD_HOME}/env3.12/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+    && source ${BUILD_HOME}/env3.12/bin/activate \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 \
+    && pip3 install --upgrade \"pip<25.0\" \"setuptools>=65.5.0,<70.0.0\" wheel \
+    && sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ${BUILD_HOME}/env3.12/lib/python3.12/site-packages/pkg_resources/__init__.py \
+    && pip3 install --no-build-isolation -r /opt/requirements.txt \
+    && pip3 freeze --user"
+
+RUN virtualenv --python=python3.13 ${BUILD_HOME}/env3.13
+RUN chmod +x ${BUILD_HOME}/env3.13/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+    && source ${BUILD_HOME}/env3.13/bin/activate \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 \
+    && pip3 install --upgrade \"pip<25.0\" \"setuptools>=65.5.0,<70.0.0\" wheel \
+    && sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ${BUILD_HOME}/env3.13/lib/python3.13/site-packages/pkg_resources/__init__.py \
+    && pip3 install --no-build-isolation -r /opt/requirements.txt \
+    && pip3 freeze --user"
+
 # 4* requires java8, sudo doesn't work on cross-platform builds
 USER root
 RUN update-alternatives --set java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java

.build/run-tests.sh

@@ -24,8 +24,6 @@
 set -o errexit
 set -o pipefail
 
-[ $DEBUG ] && set -x
-
 # variables, with defaults
 [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)"
 [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build"

.jenkins/Jenkinsfile

@@ -59,7 +59,7 @@
 /** CONSTANTS for both the pipeline and scripting **/
 import groovy.transform.Field
 @Field List<String> archsSupported   = ["amd64", "arm64"]
-@Field List<String> pythonsSupported = ["3.8", "3.11"]
+@Field List<String> pythonsSupported = ["3.8", "3.11", "3.12", "3.13"]
 @Field String       pythonDefault    = "3.8"
 /** CONSTANTS end **********************************/
 
@@ -124,7 +124,7 @@ pipeline {
   }
   post {
     failure {
-      echo "ERROR pipeline failed – not all tests were run"
+      echo "ERROR pipeline failed – not all tests were run"
     }
     always {
       sendNotifications()
@@ -254,6 +254,7 @@ def tasks() {
     (isStageEnabled(axis['step'])) && // skip disabled steps
     !(axis['python'] != pythonDefault && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test
     !(axis['cython'] != 'no' && 'cqlsh-test' != axis['step']) && // cython only for cqlsh-test, disable for others
+    !(axis['cython'] == 'yes' && (axis['python'] == '3.12' || axis['python'] == '3.13')) && // Skip cython for Python 3.12+ see CASSANDRA-21482
     !(axis['jdk'] != javaVersionDefault && ('cqlsh-test' == axis['step'] || 'simulator-dtest' == axis['step'] || axis['step'].contains('dtest-upgrade'))) && // run cqlsh-test, simulator-dtest, *dtest-upgrade only with jdk11
     // Disable splits for all but proper stages
     !(axis['split'] > 1 && !stepsMap.findAll { entry -> entry.value.splits >= axis['split'] }.keySet().contains(axis['step'])) &&

CHANGES.txt

@@ -1,4 +1,5 @@
 5.0.9
+ * Support Python 3.12 and 3.13 in cqlsh (CASSANDRA-20997)
  * Fix AssertionError in hasReplicaWithOngoingRepair when parallel_repair_count > 1 (CASSANDRA-21426)
  * putShortVolatile is not volatile in InMemoryTrie (CASSANDRA-21353)
  * Fix RequestFailureReason serializer and nits in a few others (CASSANDRA-21437)

bin/cqlsh

@@ -63,7 +63,7 @@ is_supported_version() {
     major_version="${version%.*}"
     minor_version="${version#*.}"
     # python 3.8-3.11 are supported
-    if [ "$major_version" = 3 ] && [ "$minor_version" -ge 8 ] && [ "$minor_version" -le 11 ]; then
+    if [ "$major_version" = 3 ] && [ "$minor_version" -ge 8 ] && [ "$minor_version" -le 13 ]; then
         echo "supported"
     # python 3.6-3.7 are deprecated
     elif [ "$major_version" = 3 ] && [ "$minor_version" -ge 6 ] && [ "$minor_version" -le 7 ]; then
@@ -88,7 +88,7 @@ run_if_supported_version() {
             exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" "$@"
             exit
         else
-            echo "Warning: unsupported version of Python, required 3.6-3.11 but found" "$version" >&2
+            echo "Warning: unsupported version of Python, required 3.6-3.13 but found" "$version" >&2
         fi
     fi
 }

bin/cqlsh.py

@@ -21,8 +21,8 @@
 import sys
 from glob import glob
 
-if sys.version_info < (3, 6) or sys.version_info >= (3, 12):
-    sys.exit("\ncqlsh requires Python 3.6-3.11\n")
+if sys.version_info < (3, 6) or sys.version_info >= (3, 14):
+    sys.exit("\ncqlsh requires Python 3.6-3.13\n")
 
 # see CASSANDRA-10428
 if platform.python_implementation().startswith('Jython'):
@@ -56,7 +56,7 @@ def find_zip(libprefix):
     sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
 
 # the driver needs dependencies
-third_parties = ('pure_sasl-', 'wcwidth-')
+third_parties = ('pure_sasl-', 'wcwidth-', 'pyasyncore-')
 
 for lib in third_parties:
     lib_zip = find_zip(lib)

pylib/cassandra-cqlsh-tests.sh

@@ -23,6 +23,8 @@
 #
 ################################
 
+[ $DEBUG ] && set -x
+
 WORKSPACE=$1
 
 [ "x${WORKSPACE}" != "x" ] || WORKSPACE="$(readlink -f $(dirname "$0")/..)"
@@ -63,6 +65,10 @@ rm -fr ${DIST_DIR}/venv ${DIST_DIR}/test/{html,output,logs}
 virtualenv-clone ${BUILD_HOME}/env${python_version} ${BUILD_DIR}/venv || virtualenv --python=python3 ${BUILD_DIR}/venv
 source ${BUILD_DIR}/venv/bin/activate
 
+# Force pip's legacy pkg_resources metadata backend (instead of the importlib.metadata
+# backend that pip defaults to on Python 3.11+). see ubuntu-test.docker
+export _PIP_USE_IMPORTLIB_METADATA=0
+
 pip install --exists-action w -r ${CASSANDRA_DIR}/pylib/requirements.txt
 pip freeze
 

pylib/requirements.txt

@@ -23,3 +23,4 @@
 coverage
 pytest
 wcwidth
+pyasyncore