Skip to content

Commit f33c113

Browse files
michaelsembwevermichaelsembwever
committed
Merge branch 'cassandra-5.0' into cassandra-6.0
* cassandra-5.0: Fix ci-cassandra.a.o agent workspaces for Cassandra-5.0 (and above) not being cleaned
2 parents 14ace18 + 26bdaa5 commit f33c113

1 file changed

Lines changed: 102 additions & 92 deletions

File tree

‎.jenkins/Jenkinsfile‎

Lines changed: 102 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -352,46 +352,46 @@ def build(command, cell) {
352352
nodeExclusion = "&&!${NODE_NAME}"
353353
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
354354
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
355-
fetchSource(cell.step, cell.arch, cell.jdk)
356-
sh """
357-
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
358-
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
359-
"""
360-
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['debian-build'])
361-
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
362-
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
363-
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
364-
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
365-
timeout(time: 1, unit: 'HOURS') {
366-
try {
367-
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
368-
dir("build") {
369-
archiveArtifacts artifacts: "${logfile}", fingerprint: true
370-
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
371-
}
372-
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
373-
if ("jar" == cell.step) {
374-
stash name: "${cell.arch}_${cell.jdk}"
375-
}
376-
} catch (exc) {
377-
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
378-
def descriptions = []
379-
for (def cause in exc.getCauses()) {
380-
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
381-
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
382-
throw exc // user explicitly aborted — do not retry
355+
try {
356+
fetchSource(cell.step, cell.arch, cell.jdk)
357+
sh """
358+
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
359+
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
360+
"""
361+
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['debian-build'])
362+
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
363+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
364+
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
365+
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
366+
timeout(time: 1, unit: 'HOURS') {
367+
try {
368+
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
369+
dir("build") {
370+
archiveArtifacts artifacts: "${logfile}", fingerprint: true
371+
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
372+
}
373+
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
374+
if ("jar" == cell.step) {
375+
stash name: "${cell.arch}_${cell.jdk}"
376+
}
377+
} catch (exc) {
378+
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
379+
def descriptions = []
380+
for (def cause in exc.getCauses()) {
381+
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
382+
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
383+
throw exc // user explicitly aborted — do not retry
384+
}
385+
descriptions.add(cause.getShortDescription())
383386
}
384-
descriptions.add(cause.getShortDescription())
387+
error("Retryable interruption: ${descriptions.join(', ')}")
385388
}
386-
error("Retryable interruption: ${descriptions.join(', ')}")
389+
throw exc
387390
}
388-
throw exc
389391
}
392+
} finally {
393+
cleanAgent(cell.step)
390394
}
391-
dir("build") {
392-
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
393-
}
394-
cleanAgent(cell.step)
395395
}
396396
}
397397
}
@@ -410,69 +410,72 @@ def test(command, cell) {
410410
nodeExclusion = "&&!${NODE_NAME}"
411411
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
412412
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
413-
fetchSource(cell.step, cell.arch, cell.jdk)
414-
fetchDockerImages(['ubuntu-test'])
415-
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
416-
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
417-
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
418-
script_vars = "${script_vars} python_version=\'${cell.python}\'"
419-
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
420-
if ("cqlsh-test" == cell.step) {
421-
script_vars = "${script_vars} cython=\'${cell.cython}\'"
422-
}
423-
script_vars = fetchDTestsSource(command, script_vars)
424-
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
425-
def timer = System.currentTimeMillis()
426-
try {
427-
buildJVMDTestJars(cell, script_vars, logfile)
428-
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
429-
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
430-
dir("build") {
431-
archiveArtifacts artifacts: "${logfile}", fingerprint: true
432-
}
433-
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
434-
} catch (exc) {
435-
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
436-
def descriptions = []
437-
for (def cause in exc.getCauses()) {
438-
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
439-
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
440-
throw exc // user explicitly aborted — do not retry
413+
try {
414+
fetchSource(cell.step, cell.arch, cell.jdk)
415+
fetchDockerImages(['ubuntu-test'])
416+
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
417+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
418+
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
419+
script_vars = "${script_vars} python_version=\'${cell.python}\'"
420+
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
421+
if ("cqlsh-test" == cell.step) {
422+
script_vars = "${script_vars} cython=\'${cell.cython}\'"
423+
}
424+
script_vars = fetchDTestsSource(command, script_vars)
425+
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
426+
def timer = System.currentTimeMillis()
427+
try {
428+
buildJVMDTestJars(cell, script_vars, logfile)
429+
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
430+
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
431+
dir("build") {
432+
archiveArtifacts artifacts: "${logfile}", fingerprint: true
433+
}
434+
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
435+
} catch (exc) {
436+
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
437+
def descriptions = []
438+
for (def cause in exc.getCauses()) {
439+
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
440+
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
441+
throw exc // user explicitly aborted — do not retry
442+
}
443+
descriptions.add(cause.getShortDescription())
441444
}
442-
descriptions.add(cause.getShortDescription())
445+
error("Retryable interruption: ${descriptions.join(', ')}")
443446
}
444-
error("Retryable interruption: ${descriptions.join(', ')}")
447+
throw exc
448+
} finally {
449+
def duration = System.currentTimeMillis() - timer
450+
def formattedTime = String.format("%tT.%tL", duration, duration)
451+
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
445452
}
446-
throw exc
447-
} finally {
448-
def duration = System.currentTimeMillis() - timer
449-
def formattedTime = String.format("%tT.%tL", duration, duration)
450-
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
451453
}
452-
}
453-
dir("build") {
454-
sh """
455-
mkdir -p test/output/${cell.step}
456-
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
457-
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
458-
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
459-
"""
460-
if (!cell.step.startsWith("microbench")) {
461-
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
454+
dir("build") {
455+
sh """
456+
mkdir -p test/output/${cell.step}
457+
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
458+
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
459+
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
460+
"""
461+
if (!cell.step.startsWith("microbench")) {
462+
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
463+
}
464+
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
465+
sh """
466+
echo "docker memory/oomkiller debug:"
467+
cat /sys/fs/cgroup/docker/memory.events || true
468+
"""
469+
sh """
470+
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
471+
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
472+
"""
473+
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
474+
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
462475
}
463-
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
464-
sh """
465-
echo "docker memory/oomkiller debug:"
466-
cat /sys/fs/cgroup/docker/memory.events || true
467-
"""
468-
sh """
469-
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
470-
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
471-
"""
472-
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
473-
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
476+
} finally {
477+
cleanAgent(cell.step)
474478
}
475-
cleanAgent(cell.step)
476479
}
477480
}
478481
}
@@ -566,6 +569,13 @@ def cleanAgent(job_name) {
566569
logAgentInfo(job_name, agentScriptsUrl)
567570
}
568571
cleanWs()
572+
if (isCanonical()) {
573+
// in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436)
574+
sh """#!/bin/bash
575+
set +e
576+
find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} +
577+
"""
578+
}
569579
}
570580

571581
def cleanAgentDocker(job_name, agentScriptsUrl) {

0 commit comments

Comments
 (0)