Merge branch 'cassandra-6.0' into CASSANDRA-21134-direct-compaction-writes

Author: samueldlightfoot

.build/build-bench.xml

@@ -20,6 +20,11 @@
          xmlns:if="ant:if" xmlns:unless="ant:unless">
 
     <property name="async-profiler.version" value="2.9"/>
+
+    <!-- TODO https://issues.apache.org/jira/browse/CASSANDRA-18873 -->
+    <!-- Blacklist of broken benchmarks -->
+    <property name="microbench.exclude.pattern" value="AtomicBTreePartitionUpdateBench|BTreeSearchIteratorBench|BTreeTransformBench|BTreeUpdateBench|CompactionWideRowBench|FastThreadLocalBench|KeyLookupBench|(instance.*Bench)|LatencyTrackingBench|MessageOutBench|MutationBench|ReadWriteBench|SSTablePipeCursorBench|SSTableReadingCursorBench|SSTableReadingFileBench|SSTableReadingFileCursorBench|ZeroCopyStreamingBench"/>
+
     <condition property="async-profiler.suffix" value="linux-arm64">
         <and>
             <os arch="aarch64"/>
@@ -83,6 +88,7 @@
     <macrodef name="jmh">
         <element name="extra-args" optional="true"/>
         <sequential>
+            <mkdir dir="${build.test.output.dir}"/>
             <java classname="org.openjdk.jmh.Main" fork="true" failonerror="true" >
                 <classpath>
                     <path refid="cassandra.classpath.test"/>
@@ -106,18 +112,17 @@
                 <arg value="-rf"/>
                 <arg value="json"/>
                 <arg value="-rff"/>
-                <arg value="${build.test.dir}/jmh-result.json"/>
+                <arg value="${build.test.output.dir}/jmh-result.json"/>
                 <arg value="-v"/>
                 <arg value="EXTRA"/>
 
                 <arg line="${jmh.args}"/>
 
                 <extra-args/>
 
-                <!-- TODO https://issues.apache.org/jira/browse/CASSANDRA-18873 -->
-                <!-- The classes listed below are broken, and so currently ignored, see CASSANDRA-18873 -->
+                <!-- Exclude broken, see CASSANDRA-18873 -->
                 <arg value="-e" if:blank="${benchmark.name}"/>
-                <arg value="AtomicBTreePartitionUpdateBench|BTreeSearchIteratorBench|BTreeTransformBench|BTreeUpdateBench|CompactionWideRowBench|FastThreadLocalBench|KeyLookupBench|(instance.*Bench)|LatencyTrackingBench|MessageOutBench|MutationBench|ReadWriteBench|SSTablePipeCursorBench|SSTableReadingCursorBench|SSTableReadingFileBench|SSTableReadingFileCursorBench|ZeroCopyStreamingBench" if:blank="${benchmark.name}"/>
+                <arg value="${microbench.exclude.pattern}" if:blank="${benchmark.name}"/>
 
                 <arg value=".*microbench.*${benchmark.name}"/>
             </java>

.build/docker/almalinux-build.docker

@@ -81,27 +81,6 @@ RUN sh -c '\
 # Install Antora for documentation generation
 RUN npm i -g @antora/cli@2.3 @antora/site-generator-default@2.3 @djencks/asciidoctor-openblock
 
-# install golang.  GO_VERSION_SHA must be updated with VERSION
-
-RUN sh -c '\
-  set -ex ;\
-  GO_VERSION="1.26.1" ;\
-  GO_VERSION_SHAS="031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7 65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08 353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f" ;\
-  GO_OS=linux ;\
-  [ $(uname) = "Darwin" ] && GO_OS=darwin ;\
-  GO_PLATFORM=amd64 ;\
-  [ $(uname -m) = "aarch64" ] && GO_PLATFORM=arm64 ;\
-  GO_TAR="go${GO_VERSION}.${GO_OS}-${GO_PLATFORM}.tar.gz" ;\
-  curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 3600 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
-  GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\
-  echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\
-  tar -C /usr/local -xzf $GO_TAR ;\
-  rm $GO_TAR'
-
-ENV GOROOT="/usr/local/go"
-ENV GOPATH="$BUILD_HOME/go"
-ENV PATH="$PATH:/usr/local/go/bin"
-
 # Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
 COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
 RUN alternatives --set java $(alternatives --display java | grep "family java-11-openjdk" | cut -d' ' -f1)

.build/docker/debian-build.docker

@@ -87,23 +87,6 @@ RUN sh -c '\
 # Install Antora for documentation generation
 RUN npm i -g @antora/cli@2.3 @antora/site-generator-default@2.3 @djencks/asciidoctor-openblock
 
-# install golang.  GO_VERSION_SHA must be updated with VERSION
-RUN sh -c '\
-  set -ex ;\
-  GO_VERSION="1.26.1" ;\
-  GO_VERSION_SHAS="031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7 65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08 353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f" ;\
-  GO_OS=linux ;\
-  GO_TAR="go${GO_VERSION}.${GO_OS}-$(dpkg --print-architecture).tar.gz" ;\
-  curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 3600 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
-  GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\
-  echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\
-  tar -C /usr/local -xzf $GO_TAR ;\
-  rm $GO_TAR'
-
-ENV GOROOT="/usr/local/go"
-ENV GOPATH="$BUILD_HOME/go"
-ENV PATH="$PATH:/usr/local/go/bin"
-
 # allow lower UIDs and GIDs
 RUN sed -i 's/UID_MIN			 1000/UID_MIN			 100/' /etc/login.defs
 RUN sed -i 's/UID_MIN			 1000/UID_MIN			 10/' /etc/login.defs

.build/git/install-git-defaults.sh

@@ -63,8 +63,18 @@ install_hook() {
       installed=false
     fi
   fi
+  # Resolve the source hooks directory. When git is configured with
+  # core.symlinks=false (e.g. for security reasons, or on filesystems that
+  # don't support symlinks), a symlinked hooks dir such as post-switch is
+  # checked out as a plain text file whose contents are the link target.
+  # Follow that indirection so the build works regardless of core.symlinks.
+  local src_dir="$bin/git-hooks/${name}"
+  while [[ -f "$src_dir" ]]; do
+    src_dir="$(dirname "$src_dir")/$(cat "$src_dir")"
+  done
+
   # install all hooks
-  cp "$bin"/git-hooks/"${name}"/* "$d_dir"/
+  cp "$src_dir"/* "$d_dir"/
 
   # install coordinator hook
   install_template_script "$script_name" "$d_dir"

.build/run-tests.sh

@@ -284,16 +284,86 @@ _run_testlist() {
     [ "${_test_iterations}" -eq 1 ] || printf "––––\nfailure rate: ${failures}/${_test_iterations}\n"
 }
 
+_list_microbench_tests() {
+    # Extract blacklist from build-bench.xml property (see CASSANDRA-18873)
+    local blacklist_pattern=$(grep 'name="microbench.exclude.pattern"' .build/build-bench.xml | sed -n 's/.*value="\([^"]*\)".*/\1/p')
+
+    # Find all *Bench.java files, strip prefix, sort, and filter out blacklisted ones
+    find "test/microbench" -name '*Bench.java' | \
+        sed "s;^test/microbench/;;g" | \
+        sort | \
+        grep -vE "(${blacklist_pattern})\.java$"
+}
+
+_run_microbench() {
+    local _target=$1
+    local _test_name_regexp=$2
+    local _split_chunk=$3
+    local testlist=""
+
+    # Assert no *Test.java files exist under test/microbench
+    # uncomment once CachingBenchTest and GcCompactionBenchTest are rewritten to JMH benchmarks
+    #_list_tests "microbench" | grep -q 'Test\.java$' && error 1 "Found *Test.java files under test/microbench, these should be moved to test/unit"
+
+    # Build test list from either regexp or split
+    if [ -n "${_test_name_regexp}" ]; then
+      echo "Running tests: ${_test_name_regexp}"
+      # test regexp can come in csv
+      for i in ${_test_name_regexp//,/ }; do
+        [ -n "${testlist}" ] && testlist="${testlist}"$'\n'
+        testlist="${testlist}$( _list_microbench_tests | _split_tests "${i}")"
+      done
+      [[ -z "${testlist}" ]] && error 1 "No tests found in test name regexp: ${_test_name_regexp}"
+    else
+      [ -n "${_split_chunk}" ] || { error 1 "Neither name regexp or split chunk defined"; }
+      echo "Running split: ${_split_chunk}"
+      testlist="$( _list_microbench_tests | _split_tests "${_split_chunk}")"
+      if [[ -z "${testlist}" ]]; then
+        echo "No microbench tests in split ${_split_chunk}, skipping"
+        return 0
+      fi
+    fi
+
+    # Convert file paths to the JMH classname pattern
+    local benchmark_pattern=$(echo "${testlist}" | sed 's/\.java$//g' | sed 's|^org/apache/cassandra/test/microbench/||g' | sed 's/\//./g' | tr '\n' '|' | sed 's/|$//')
+    echo "Running benchmarks: ${benchmark_pattern}"
+
+    # override build.test.output.dir, adding jdk and arch to output path for report separation
+    local -r java_version="$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')"
+    local -r arch="$(uname -m)"
+    local -r output_dir="${DIST_DIR}/test/output/${_target}/jdk${java_version}/${arch}/${_split_chunk//\//_}"
+
+    ant $_target ${ANT_TEST_OPTS} -Dbuild.test.output.dir=${output_dir} -Dbenchmark.name="${benchmark_pattern}" -Dmaven.test.failure.ignore=true
+
+    # Post-process jmh-result.json to add jdk and arch parameters
+    local jmh_result="${output_dir}/jmh-result.json"
+    if [ -f "${jmh_result}" ]; then
+        python3 -c "
+import json,sys
+with open('${jmh_result}','r') as f:
+    data=json.load(f)
+for r in (data if isinstance(data,list) else [data]):
+    if 'params' not in r:
+        r['params']={}
+    r['params']['jdk']='${java_version}'
+    r['params']['arch']='${arch}'
+with open('${jmh_result}','w') as f:
+    json.dump(data,f)
+"
+    fi
+}
+
 _main() {
   # parameters
   local -r target="${test_target/-repeat/}"
-  local -r split_chunk="${chunk:-'1/1'}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
+  local -r split_chunk="${chunk:-1/1}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
 
   # check split_chunk is compatible with target (if not a regexp)
   if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
     case ${target} in
-      "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "cqlsh-test")
-          error 1 "Target ${target} does not support splits."
+      "stress-test" | "fqltool-test" | "cqlsh-test" | "sstableloader-test")
+          echo "Target ${target} does not support splits."
+          exit 1
           ;;
         *)
           ;;
@@ -357,8 +427,7 @@ _main() {
       ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
       ;;
     "microbench" | "microbench-test")
-      [[ "x${test_name_regexp}" != "x" ]] && test_name_regexp="-Dbenchmark.name=${test_name_regexp}"
-      ant $target ${ANT_TEST_OPTS} ${test_name_regexp} -Dmaven.test.failure.ignore=true
+      _run_microbench "$target" "${test_name_regexp}" "${split_chunk}"
       ;;
     "test")
       _run_testlist "unit" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"

.jenkins/Jenkinsfile

@@ -216,9 +216,9 @@ def tasks() {
     'dtest-upgrade-novnode': [splits: 160, size: 'large'],
     'dtest-upgrade-large': [splits: 40, size: 'large'],
     'dtest-upgrade-large-novnode': [splits: 40, size: 'large'],
-    'microbench-test': [splits: 1, size: 'large', timeout_hours: 2], // TODO implement splits
+    'microbench-test': [splits: 4, size: 'large', timeout_hours: 2],
     // performance tests need 'cassandra-*large-dedicated' nodes
-    'microbench': [splits: 1, size: 'large', timeout_hours: 6, benchmark: true],
+    'microbench': [splits: 4, size: 'large', timeout_hours: 6, benchmark: true],
   ]
   testSteps.each() {
     it.value.put('type', 'test')
@@ -470,8 +470,8 @@ def test(command, cell) {
                   find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
                   echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
                 """
-              archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
-              copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
+              archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true
+              copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
             }
           } finally {
             cleanAgent(cell.step)
@@ -619,9 +619,10 @@ def generateTestReports() {
           unzip -x -d build/test -q output.zip ) ${teeSuffix}
          """
     } else {
-      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
     }
-    if (fileExists('build/test/output')) {
+    // merge and summarise test reports
+    if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name "*.xml.xz" -print -quit)"', returnStatus: true) == 0) {
       // merge splits for each target's test report, other axes are kept separate
       //   TODO parallelised for loop
       //   TODO results_details.tar.xz needs to include all logs for failed tests
@@ -647,11 +648,31 @@ def generateTestReports() {
 
       dir('build/') {
         archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
-        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile},test/jmh-result.json')
+        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
       }
     }
-    if (fileExists('build/test/jmh-result.json')) {
-      jmhReport('build/test/jmh-result.json')
+    processJmhReports()
+  }
+}
+
+def processJmhReports() {
+  if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name jmh-result.json -print -quit)"', returnStatus: true) == 0) {
+    sh ''' python3 -c "
+import glob,json
+m=[]
+for f in glob.glob('build/test/output/**/jmh-result.json',recursive=True):
+  d=json.load(open(f))
+  m.extend(d if isinstance(d,list) else [d])
+o=open('build/test/output/combined-result.json','w')
+json.dump(m,o)
+o.close()
+print(f'combined {len(m)} results')
+"
+  '''
+    jmhReport('build/test/output/combined-result.json')
+    dir('build/') {
+      archiveArtifacts artifacts: "test/output/combined-result.json", fingerprint: true
+      copyToNightlies('test/output/combined-result.json')
     }
   }
 }

CHANGES.txt

@@ -1,5 +1,19 @@
 6.0-alpha2
  * Support direct I/O for background SSTable writes (CASSANDRA-21134)
+ * Report cancelled read command execution to coordinator as a RequestFailure.TIMEOUT (CASSANDRA-21468)
+ * Fix TCM log catchup from peer with snapshots and gaps in the log sequence (CASSANDRA-21455)
+ * Speed up nodetool doc generation by producing all command help in a single jvm (CASSANDRA-21444)
+ * Always send TCM commit failures as Messaging failures (CASSANDRA-21457)
+ * Fix ReadCommand serializedSize() using incorrect epoch (CASSANDRA-21438)
+ * Allocation improvements in ProtocolVersion, StorageProxy and MerkleTree (CASSANDRA-21199)
+ * Don’t leave autocompaction disabled during bootstrap and replace (CASSANDRA-21236)
+ * Make nodetool abortbootstrap more robust (CASSANDRA-21235)
+ * Don't clear prepared statement cache on nodetool cms initialize (CASSANDRA-21234)
+ * Improve performance when deserializing cluster metadata  (CASSANDRA-21224)
+ * Minor TokenMap performance improvement (CASSANDRA-21223)
+ * Handle lost response when committing PrepareMove (CASSANDRA-21222)
+ * SEPExecutor.maybeExecuteImmediately does not always execute tasks immediately despite available worker capacity (CASSANDRA-21429)
+ * Safely regain ranges and delete retired command stores (CASSANDRA-21212)
  * Reduce memory allocations in miscellaneous places along read path (CASSANDRA-21360)
  * Avoid ByteBuffer wrapping in cql3.selection.Selector.InputRow to reduce memory allocation rate (CASSANDRA-21362)
  * Reduce cost to calculate BTreeRow.minDeletionTime (CASSANDRA-21414)
@@ -40,13 +54,21 @@
  * Fix a removed TTLed row re-appearance in a materialized view after a cursor compaction (CASSANDRA-21152)
  * Rework ZSTD dictionary compression logic to create a trainer per training (CASSANDRA-21209)
 Merged from 5.0:
+ * putShortVolatile is not volatile in InMemoryTrie (CASSANDRA-21353)
+ * Fix RequestFailureReason serializer and nits in a few others (CASSANDRA-21437)
+ * Remove golang dependency in gen-doc and replace with python implementation (CASSANDRA-21432)
  * Use estimated compressed size for tables to check if there is enough free space for a compaction (CASSANDRA-21245)
  * Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348)
  * Ensure SAI sends range tombstones to the coordinator for queries on static columns (CASSANDRA-21332)
 Merged from 4.1:
  * Add Paxos v2 option and informatin in cassandra.yaml (CASSANDRA-21316)
  * Harden data resurrection startup check with atomic heartbeat file write with fallback (CASSANDRA-21290)
 Merged from 4.0:
+ * Rename conflicting nodetool import --copy-data short option from -p to -cd (CASSANDRA-20214)
+ * Fix PasswordObfuscator failing to obfuscate certain passwords (CASSANDRA-21113)
+ * Fix negative memtable allocator ownership when an update is shadowed by an existing row deletion (CASSANDRA-21469)
+ * Consider first token of SSTable when calculating SSTable intersection in LeveledScanner  (CASSANDRA-21369)
+ * Remove inFlightEcho entry on ECHO_REQ failure (CASSANDRA-21428)
  * Validate snapshot names (CASSANDRA-21389)
  * BTree.FastBuilder.reset() fails to clear savedBuffer and savedNextKey, causing ClassCastException and SSTable header corruption during schema disagreement (CASSANDRA-21216, CASSANDRA-21260)
  * Backport CASSANDRA-17810 fix and improve RTBoundValidator error messages (CASSANDRA-18282)

build.xml

@@ -605,7 +605,29 @@
         </wikitext-to-html>
     </target>
 
-    <target name="gen-asciidoc" description="Generate dynamic asciidoc pages" depends="jar" unless="ant.gen-doc.skip">
+    <target name="gen-nodetool-text" description="Generate nodetool command help text files consumed by the docs" depends="jar" unless="ant.gen-doc.skip">
+        <property name="nodetool.help.outdir" location="${doc.dir}/modules/cassandra/examples/TEXT/NODETOOL"/>
+        <property name="nodetool.help.classes" location="${build.dir}/nodetool-helpgen"/>
+        <mkdir dir="${nodetool.help.outdir}"/>
+        <mkdir dir="${nodetool.help.classes}"/>
+        <javac srcdir="${test.unit.src}" includes="org/apache/cassandra/tools/nodetool/NodetoolHelpGenerator.java"
+               destdir="${nodetool.help.classes}" includeantruntime="false" classpathref="cassandra.classpath"/>
+        <java classname="org.apache.cassandra.tools.nodetool.NodetoolHelpGenerator" fork="true" failonerror="true">
+            <classpath>
+                <pathelement location="${nodetool.help.classes}"/>
+                <pathelement location="${basedir}/conf"/>
+                <path refid="cassandra.classpath"/>
+            </classpath>
+            <jvmarg line="${java-jvmargs}"/>
+            <sysproperty key="logback.configurationFile" value="logback-tools.xml"/>
+            <sysproperty key="cassandra.cli.layout" value="airline"/>
+            <arg value="--dir"/>
+            <arg value="${nodetool.help.outdir}"/>
+            <arg value="--txt"/>
+        </java>
+    </target>
+
+    <target name="gen-asciidoc" description="Generate dynamic asciidoc pages" depends="jar,gen-nodetool-text" unless="ant.gen-doc.skip">
         <exec executable="make" osfamily="unix" dir="${doc.dir}" failonerror="true">
             <arg value="gen-asciidoc"/>
         </exec>
@@ -1135,7 +1157,7 @@
           <exclude name="tools/bin/*.in.sh" />
           <include name=".build/**/*.sh"/>
           <include name="${accord.dir}/gradlew" />
-          <include name="doc/scripts/process-native-protocol-specs-in-docker.sh"/>
+          <include name="doc/scripts/gen-native-protocol-docs.sh"/>
         </tarfileset>
       </tar>