Skip to content

Commit 26bdaa5

Browse files
michaelsembwevermichaelsembwever
committed
Fix ci-cassandra.a.o agent workspaces for Cassandra-5.0 (and above) not being cleaned
The build+test cells run in `ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}")`, so every build writes a new per-build directory. The `cleanWs()` only cleans the current cell's leaf, and it sat as the last statement in the `ws` block — so any cell hitting error() (failed/timed-out/aborted) exits before reaching it and leaves the full cell's workspace build behind. patch by Mick Semb Wever; reviewed by Dmitry Konstantinov for CASSANDRA-20436
1 parent b78f654 commit 26bdaa5

1 file changed

Lines changed: 105 additions & 92 deletions

File tree

‎.jenkins/Jenkinsfile‎

Lines changed: 105 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -349,46 +349,49 @@ def build(command, cell) {
349349
nodeExclusion = "&&!${NODE_NAME}"
350350
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
351351
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
352-
fetchSource(cell.step, cell.arch, cell.jdk)
353-
sh """
354-
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
355-
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
356-
"""
357-
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
358-
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
359-
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
360-
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
361-
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
362-
timeout(time: 1, unit: 'HOURS') {
363-
try {
364-
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
365-
dir("build") {
366-
archiveArtifacts artifacts: "${logfile}", fingerprint: true
367-
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
368-
}
369-
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
370-
if ("jar" == cell.step) {
371-
stash name: "${cell.arch}_${cell.jdk}"
372-
}
373-
} catch (exc) {
374-
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
375-
def descriptions = []
376-
for (def cause in exc.getCauses()) {
377-
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
378-
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
379-
throw exc // user explicitly aborted — do not retry
352+
try {
353+
fetchSource(cell.step, cell.arch, cell.jdk)
354+
sh """
355+
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
356+
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
357+
"""
358+
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
359+
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
360+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
361+
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
362+
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
363+
timeout(time: 1, unit: 'HOURS') {
364+
try {
365+
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
366+
dir("build") {
367+
archiveArtifacts artifacts: "${logfile}", fingerprint: true
368+
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
369+
}
370+
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
371+
if ("jar" == cell.step) {
372+
stash name: "${cell.arch}_${cell.jdk}"
373+
}
374+
dir("build") {
375+
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
376+
}
377+
} catch (exc) {
378+
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
379+
def descriptions = []
380+
for (def cause in exc.getCauses()) {
381+
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
382+
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
383+
throw exc // user explicitly aborted — do not retry
384+
}
385+
descriptions.add(cause.getShortDescription())
380386
}
381-
descriptions.add(cause.getShortDescription())
387+
error("Retryable interruption: ${descriptions.join(', ')}")
382388
}
383-
error("Retryable interruption: ${descriptions.join(', ')}")
389+
throw exc
384390
}
385-
throw exc
386391
}
392+
} finally {
393+
cleanAgent(cell.step)
387394
}
388-
dir("build") {
389-
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
390-
}
391-
cleanAgent(cell.step)
392395
}
393396
}
394397
}
@@ -407,69 +410,72 @@ def test(command, cell) {
407410
nodeExclusion = "&&!${NODE_NAME}"
408411
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
409412
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
410-
fetchSource(cell.step, cell.arch, cell.jdk)
411-
fetchDockerImages(['ubuntu-test'])
412-
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
413-
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
414-
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
415-
script_vars = "${script_vars} python_version=\'${cell.python}\'"
416-
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
417-
if ("cqlsh-test" == cell.step) {
418-
script_vars = "${script_vars} cython=\'${cell.cython}\'"
419-
}
420-
script_vars = fetchDTestsSource(command, script_vars)
421-
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
422-
def timer = System.currentTimeMillis()
423-
try {
424-
buildJVMDTestJars(cell, script_vars, logfile)
425-
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
426-
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
427-
dir("build") {
428-
archiveArtifacts artifacts: "${logfile}", fingerprint: true
429-
}
430-
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
431-
} catch (exc) {
432-
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
433-
def descriptions = []
434-
for (def cause in exc.getCauses()) {
435-
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
436-
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
437-
throw exc // user explicitly aborted — do not retry
413+
try {
414+
fetchSource(cell.step, cell.arch, cell.jdk)
415+
fetchDockerImages(['ubuntu-test'])
416+
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
417+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
418+
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
419+
script_vars = "${script_vars} python_version=\'${cell.python}\'"
420+
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
421+
if ("cqlsh-test" == cell.step) {
422+
script_vars = "${script_vars} cython=\'${cell.cython}\'"
423+
}
424+
script_vars = fetchDTestsSource(command, script_vars)
425+
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
426+
def timer = System.currentTimeMillis()
427+
try {
428+
buildJVMDTestJars(cell, script_vars, logfile)
429+
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
430+
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
431+
dir("build") {
432+
archiveArtifacts artifacts: "${logfile}", fingerprint: true
433+
}
434+
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
435+
} catch (exc) {
436+
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
437+
def descriptions = []
438+
for (def cause in exc.getCauses()) {
439+
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
440+
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
441+
throw exc // user explicitly aborted — do not retry
442+
}
443+
descriptions.add(cause.getShortDescription())
438444
}
439-
descriptions.add(cause.getShortDescription())
445+
error("Retryable interruption: ${descriptions.join(', ')}")
440446
}
441-
error("Retryable interruption: ${descriptions.join(', ')}")
447+
throw exc
448+
} finally {
449+
def duration = System.currentTimeMillis() - timer
450+
def formattedTime = String.format("%tT.%tL", duration, duration)
451+
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
442452
}
443-
throw exc
444-
} finally {
445-
def duration = System.currentTimeMillis() - timer
446-
def formattedTime = String.format("%tT.%tL", duration, duration)
447-
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
448453
}
449-
}
450-
dir("build") {
451-
sh """
452-
mkdir -p test/output/${cell.step}
453-
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
454-
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
455-
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
456-
"""
457-
if (!cell.step.startsWith("microbench")) {
458-
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
454+
dir("build") {
455+
sh """
456+
mkdir -p test/output/${cell.step}
457+
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
458+
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
459+
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
460+
"""
461+
if (!cell.step.startsWith("microbench")) {
462+
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
463+
}
464+
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
465+
sh """
466+
echo "docker memory/oomkiller debug:"
467+
cat /sys/fs/cgroup/docker/memory.events || true
468+
"""
469+
sh """
470+
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
471+
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
472+
"""
473+
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
474+
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
459475
}
460-
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
461-
sh """
462-
echo "docker memory/oomkiller debug:"
463-
cat /sys/fs/cgroup/docker/memory.events || true
464-
"""
465-
sh """
466-
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
467-
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
468-
"""
469-
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
470-
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
476+
} finally {
477+
cleanAgent(cell.step)
471478
}
472-
cleanAgent(cell.step)
473479
}
474480
}
475481
}
@@ -563,6 +569,13 @@ def cleanAgent(job_name) {
563569
logAgentInfo(job_name, agentScriptsUrl)
564570
}
565571
cleanWs()
572+
if (isCanonical()) {
573+
// in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436)
574+
sh """#!/bin/bash
575+
set +e
576+
find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} +
577+
"""
578+
}
566579
}
567580

568581
def cleanAgentDocker(job_name, agentScriptsUrl) {

0 commit comments

Comments
 (0)