From d26662a45ae3e146b359d9aef92de605e9285d13 Mon Sep 17 00:00:00 2001 From: jorgee Date: Mon, 9 Mar 2026 14:48:31 +0100 Subject: [PATCH 1/9] recover task history when cached to ensure tries in hash are the same when task previously aborted or retired Signed-off-by: jorgee --- .../main/groovy/nextflow/processor/TaskProcessor.groovy | 8 +++++++- .../src/main/groovy/nextflow/processor/TaskRun.groovy | 5 +++++ .../src/main/groovy/nextflow/trace/TraceRecord.groovy | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index fc16c0c0a8..f0cf637267 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -796,7 +796,7 @@ class TaskProcessor { @CompileStatic final protected void checkCachedOrLaunchTask( TaskRun task, HashCode hash, boolean shouldTryCache ) { - int tries = task.failCount +1 + int tries = task.failCount + task.abortedCount + 1 while( true ) { hash = HashBuilder.defaultHasher().putBytes(hash.asBytes()).putInt(tries).hash() @@ -812,6 +812,12 @@ class TaskProcessor { final cached = shouldTryCache && exists && entry.trace.isCompleted() && checkCachedOutput(task.clone(), resumeDir, hash, entry) if( cached ) break + // #6448 When not cached but there is an entry whose status is ABORT or FAILED, the task counters for these status must be incremented to be sure future retries takes it into account to calculate the hash. + if( entry?.trace?.isFailed() ) { + task.failCount += 1 + task.config.attempt = task.failCount + 1 + } + if( entry?.trace?.isAborted() ) task.abortedCount+=1 } catch (Throwable t) { log.warn1("[${safeTaskName(task)}] Unable to resume cached task -- See log file for details", causedBy: t) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 2213783bfe..c0b96ef65e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -327,6 +327,11 @@ class TaskRun implements Cloneable { */ volatile int failCount + /** + * The number of times the execution of the task has been aborted + */ + volatile int abortedCount + /** * The number of times the submit of the task has been retried */ diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy index e5a6984643..0f854e472a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy @@ -599,6 +599,14 @@ class TraceRecord implements Serializable { store.status == 'COMPLETED' } + boolean isAborted() { + store.status == 'ABORTED' + } + + boolean isFailed() { + store.status == 'FAILED' + } + String getExecutorName() { return executorName } From ec523b6ce8b158b4c40b65dc11dfee64dc02b3c1 Mon Sep 17 00:00:00 2001 From: jorgee Date: Tue, 10 Mar 2026 09:20:32 +0100 Subject: [PATCH 2/9] manage attempts and add a test Signed-off-by: jorgee --- .../nextflow/processor/TaskProcessor.groovy | 16 +++-- .../resume-retried-with-abort.nf/.checks | 34 ++++++++++ tests/resume-retried-with-abort.nf | 65 +++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 tests/checks/resume-retried-with-abort.nf/.checks create mode 100644 tests/resume-retried-with-abort.nf diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index f0cf637267..039207df9b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -812,11 +812,10 @@ class TaskProcessor { final cached = shouldTryCache && exists && entry.trace.isCompleted() && checkCachedOutput(task.clone(), resumeDir, hash, entry) if( cached ) break - // #6448 When not cached but there is an entry whose status is ABORT or FAILED, the task counters for these status must be incremented to be sure future retries takes it into account to calculate the hash. - if( entry?.trace?.isFailed() ) { - task.failCount += 1 - task.config.attempt = task.failCount + 1 - } + + // Issue #6884: https://github.com/nextflow-io/nextflow/issues/6884 + // When not cached but there is an entry whose status is ABORT or FAILED, the task counters for these status must be incremented to be sure future retries takes it into account to calculate the hash. + if( entry?.trace?.isFailed() ) task.failCount += 1 if( entry?.trace?.isAborted() ) task.abortedCount+=1 } catch (Throwable t) { @@ -843,7 +842,12 @@ class TaskProcessor { finally { lock.release() } - + // Issue #6884: https://github.com/nextflow-io/nextflow/issues/6884 + // When cached tasks include failures and aborts and not cached. The task.attempt could not be sync with cached failures. We update the task.attempts and task.scripts + if( task.failCount > 0 && task.config.getAttempt() != task.failCount + 1 ) { + task.config.attempt = task.failCount + 1 + task.resolve(taskBody) + } // submit task for execution submitTask( task, hash, workDir ) break diff --git a/tests/checks/resume-retried-with-abort.nf/.checks b/tests/checks/resume-retried-with-abort.nf/.checks new file mode 100644 index 0000000000..11c9e6cdeb --- /dev/null +++ b/tests/checks/resume-retried-with-abort.nf/.checks @@ -0,0 +1,34 @@ +set -e + +# +# run with abort +# +echo '' +timeout --signal=INT 3 $NXF_RUN | tee stdout || true + +[[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false + + +# +# RESUME mode with fail and abort +# +echo '' +timeout --signal=INT 9 $NXF_RUN -resume | tee stdout || true + +[[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false +[[ `< .nextflow.log grep -c 'Re-submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false +# +# RESUME mode with completed and abort +# +echo '' +timeout --signal=INT 9 $NXF_RUN -resume | tee stdout || true + +[[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false + +# +# RESUME mode with cached and finish +# +echo '' +$NXF_RUN -resume | tee stdout + +[[ `< .nextflow.log grep -c 'Cached process > SMALL_SLEEP_RETRY'` == 1 ]] || false diff --git a/tests/resume-retried-with-abort.nf b/tests/resume-retried-with-abort.nf new file mode 100644 index 0000000000..89e260d3a7 --- /dev/null +++ b/tests/resume-retried-with-abort.nf @@ -0,0 +1,65 @@ +#!/usr/bin/env nextflow +/* + * Copyright 2013-2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +process LONG_SLEEP { + tag "long-sleep" + + input: + val x + + output: + stdout + + script: + ''' + echo "LONG_SLEEP start" + sleep 30 + echo "LONG_SLEEP done" + ''' +} + +process SMALL_SLEEP_RETRY { + tag "small-sleep-retry" + + errorStrategy 'retry' + maxRetries 1 + + input: + val x + + output: + stdout + + script: + """ + echo "SMALL_SLEEP_RETRY attempt: ${task.attempt}" + sleep 5 + + if [[ ${task.attempt} -eq 1 ]]; then + echo "Failing first attempt on purpose" + exit 1 + fi + + echo "Second attempt succeeded" + """ +} + +workflow { + ch = Channel.of(1) + LONG_SLEEP(ch) + SMALL_SLEEP_RETRY(ch) +} From 7ee0159af27e755b5477414b6f84aa5ba9fcec19 Mon Sep 17 00:00:00 2001 From: jorgee Date: Tue, 10 Mar 2026 14:00:35 +0100 Subject: [PATCH 3/9] increasing timeouts Signed-off-by: jorgee --- tests/checks/resume-retried-with-abort.nf/.checks | 6 +++--- tests/resume-retried-with-abort.nf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/checks/resume-retried-with-abort.nf/.checks b/tests/checks/resume-retried-with-abort.nf/.checks index 11c9e6cdeb..ca50a96738 100644 --- a/tests/checks/resume-retried-with-abort.nf/.checks +++ b/tests/checks/resume-retried-with-abort.nf/.checks @@ -4,7 +4,7 @@ set -e # run with abort # echo '' -timeout --signal=INT 3 $NXF_RUN | tee stdout || true +timeout --signal=INT 6 $NXF_RUN | tee stdout || true [[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false @@ -13,7 +13,7 @@ timeout --signal=INT 3 $NXF_RUN | tee stdout || true # RESUME mode with fail and abort # echo '' -timeout --signal=INT 9 $NXF_RUN -resume | tee stdout || true +timeout --signal=INT 12 $NXF_RUN -resume | tee stdout || true [[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false [[ `< .nextflow.log grep -c 'Re-submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false @@ -21,7 +21,7 @@ timeout --signal=INT 9 $NXF_RUN -resume | tee stdout || true # RESUME mode with completed and abort # echo '' -timeout --signal=INT 9 $NXF_RUN -resume | tee stdout || true +timeout --signal=INT 12 $NXF_RUN -resume | tee stdout || true [[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false diff --git a/tests/resume-retried-with-abort.nf b/tests/resume-retried-with-abort.nf index 89e260d3a7..c4328a7215 100644 --- a/tests/resume-retried-with-abort.nf +++ b/tests/resume-retried-with-abort.nf @@ -47,7 +47,7 @@ process SMALL_SLEEP_RETRY { script: """ echo "SMALL_SLEEP_RETRY attempt: ${task.attempt}" - sleep 5 + sleep 7 if [[ ${task.attempt} -eq 1 ]]; then echo "Failing first attempt on purpose" From a8ede0bebf04cce79a4ff42ec5e707805d9485b2 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 15 Apr 2026 14:56:00 -0500 Subject: [PATCH 4/9] cleanup, add comments Signed-off-by: Ben Sherman --- .../nextflow/processor/TaskProcessor.groovy | 53 +++++++++-------- .../resume-retried-with-abort.nf/.checks | 2 +- tests/resume-retried-with-abort.nf | 59 ++++++++----------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 51a9c938f2..d590ef143f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -15,8 +15,6 @@ */ package nextflow.processor -import static nextflow.processor.ErrorStrategy.* - import java.nio.file.FileSystems import java.nio.file.Path import java.util.concurrent.atomic.AtomicBoolean @@ -801,35 +799,46 @@ class TaskProcessor { int tries = task.failCount + task.abortedCount + 1 while( true ) { + // increment the task hash based on the "try count" + // in order to produce a deterministic sequence of + // task hashes for subsequent attempts hash = HashBuilder.defaultHasher().putBytes(hash.asBytes()).putInt(tries).hash() Path resumeDir = null boolean exists = false try { + // check for a cache entry and task directory for the given task hash final entry = session.cache.getTaskEntry(hash, this) resumeDir = entry ? FileHelper.asPath(entry.trace.getWorkDir()) : null if( resumeDir ) exists = resumeDir.exists() + // resume the task if it completed successfully and outputs are present log.trace "[${safeTaskName(task)}] Cacheable folder=${resumeDir?.toUriString()} -- exists=$exists; try=$tries; shouldTryCache=$shouldTryCache; entry=$entry" final cached = shouldTryCache && exists && entry.trace.isCompleted() && checkCachedOutput(task.clone(), resumeDir, hash, entry) if( cached ) break - // Issue #6884: https://github.com/nextflow-io/nextflow/issues/6884 - // When not cached but there is an entry whose status is ABORT or FAILED, the task counters for these status must be incremented to be sure future retries takes it into account to calculate the hash. - if( entry?.trace?.isFailed() ) task.failCount += 1 - if( entry?.trace?.isAborted() ) task.abortedCount+=1 + // if the task execution failed or was aborted, increment the + // try count so that any subsequent successful execution + // can be recovered + if( entry.trace.isFailed() ) task.failCount += 1 + if( entry.trace.isAborted() ) task.abortedCount += 1 } catch (Throwable t) { log.warn1("[${safeTaskName(task)}] Unable to resume cached task -- See log file for details", causedBy: t) } + // if the task directory exists but was not resumed, it is a + // failed or invalid execution -- increment the try count and + // check the next task hash if( exists ) { tries++ continue } + // if the task directory does not exist, execute the task + // with this task hash final lock = lockManager.acquire(hash) final workDir = task.getWorkDirFor(hash) try { @@ -845,12 +854,7 @@ class TaskProcessor { finally { lock.release() } - // Issue #6884: https://github.com/nextflow-io/nextflow/issues/6884 - // When cached tasks include failures and aborts and not cached. The task.attempt could not be sync with cached failures. We update the task.attempts and task.scripts - if( task.failCount > 0 && task.config.getAttempt() != task.failCount + 1 ) { - task.config.attempt = task.failCount + 1 - task.resolve(taskBody) - } + // submit task for execution submitTask( task, hash, workDir ) break @@ -1039,12 +1043,11 @@ class TaskProcessor { * @param task The {@code TaskRun} instance that raised an error * @param error The error object * @return - * Either a value of value of {@link ErrorStrategy} representing the error strategy chosen - * or an instance of {@TaskFault} representing the cause of the error (that implicitly means - * a {@link ErrorStrategy#TERMINATE}) + * Either an {@link ErrorStrategy} representing the selected error strategy, + * or a {@link TaskFault} representing the cause of the error (implies {@link ErrorStrategy#TERMINATE}) */ @PackageScope - final synchronized resumeOrDie( TaskRun task, Throwable error, TraceRecord traceRecord = null) { + final synchronized Object resumeOrDie( TaskRun task, Throwable error, TraceRecord traceRecord = null) { log.debug "Handling unexpected condition for\n task: name=${safeTaskName(task)}; work-dir=${task?.workDirStr}\n error [${error?.class?.name}]: ${error?.getMessage()?:error}" ErrorStrategy errorStrategy = TERMINATE @@ -1097,9 +1100,9 @@ class TaskProcessor { errorStrategy = checkErrorStrategy(task, error, taskErrCount, procErrCount, submitRetries) if( errorStrategy.soft ) { def msg = "[$task.hashLog] NOTE: ${submitTimeout ? submitErrMsg : error.message}" - if( errorStrategy == IGNORE ) + if( errorStrategy == ErrorStrategy.IGNORE ) msg += " -- Error is ignored" - else if( errorStrategy == RETRY ) + else if( errorStrategy == ErrorStrategy.RETRY ) msg += " -- Execution is retried (${submitTimeout ? submitRetries : taskErrCount})" log.info msg task.failed = true @@ -1170,12 +1173,12 @@ class TaskProcessor { } // IGNORE strategy -- just continue - if( action == IGNORE ) { - return IGNORE + if( action == ErrorStrategy.IGNORE ) { + return ErrorStrategy.IGNORE } // RETRY strategy -- check that process do not exceed 'maxError' and the task do not exceed 'maxRetries' - if( action == RETRY ) { + if( action == ErrorStrategy.RETRY ) { final int maxErrors = task.config.getMaxErrors() final int maxRetries = task.config.getMaxRetries() @@ -1194,7 +1197,7 @@ class TaskProcessor { session.abort(e) } } as Runnable) - return RETRY + return ErrorStrategy.RETRY } return TERMINATE @@ -1941,9 +1944,13 @@ class TaskProcessor { * and binding output values accordingly * * @param task The {@code TaskRun} instance to finalize + * @return + * Either an {@link ErrorStrategy} representing the selected error strategy, + * a {@link TaskFault} representing the cause of the error (implies {@link ErrorStrategy#TERMINATE}), + * or null if the task completed successfully. */ @PackageScope - final finalizeTask( TaskHandler handler) { + final Object finalizeTask( TaskHandler handler) { def task = handler.task log.trace "finalizing process > ${safeTaskName(task)} -- $task" diff --git a/tests/checks/resume-retried-with-abort.nf/.checks b/tests/checks/resume-retried-with-abort.nf/.checks index ca50a96738..e2cf1ab126 100644 --- a/tests/checks/resume-retried-with-abort.nf/.checks +++ b/tests/checks/resume-retried-with-abort.nf/.checks @@ -8,7 +8,6 @@ timeout --signal=INT 6 $NXF_RUN | tee stdout || true [[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false - # # RESUME mode with fail and abort # @@ -17,6 +16,7 @@ timeout --signal=INT 12 $NXF_RUN -resume | tee stdout || true [[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false [[ `< .nextflow.log grep -c 'Re-submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false + # # RESUME mode with completed and abort # diff --git a/tests/resume-retried-with-abort.nf b/tests/resume-retried-with-abort.nf index c4328a7215..cdff53f4ce 100644 --- a/tests/resume-retried-with-abort.nf +++ b/tests/resume-retried-with-abort.nf @@ -16,50 +16,43 @@ */ process LONG_SLEEP { - tag "long-sleep" + tag "long-sleep" - input: - val x + output: + stdout - output: - stdout - - script: - ''' - echo "LONG_SLEEP start" - sleep 30 - echo "LONG_SLEEP done" - ''' + script: + ''' + echo "LONG_SLEEP start" + sleep 30 + echo "LONG_SLEEP done" + ''' } process SMALL_SLEEP_RETRY { - tag "small-sleep-retry" - - errorStrategy 'retry' - maxRetries 1 + tag "small-sleep-retry" - input: - val x + errorStrategy 'retry' + maxRetries 1 - output: - stdout + output: + stdout - script: - """ - echo "SMALL_SLEEP_RETRY attempt: ${task.attempt}" - sleep 7 + script: + """ + echo "SMALL_SLEEP_RETRY attempt: ${task.attempt}" + sleep 7 - if [[ ${task.attempt} -eq 1 ]]; then - echo "Failing first attempt on purpose" - exit 1 - fi + if [[ ${task.attempt} -eq 1 ]]; then + echo "Failing first attempt on purpose" + exit 1 + fi - echo "Second attempt succeeded" - """ + echo "Second attempt succeeded" + """ } workflow { - ch = Channel.of(1) - LONG_SLEEP(ch) - SMALL_SLEEP_RETRY(ch) + LONG_SLEEP() + SMALL_SLEEP_RETRY() } From 703321be05d2fe9a28b86d3621ed5a5c6626af12 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 15 Apr 2026 15:05:36 -0500 Subject: [PATCH 5/9] Fix failing tests Signed-off-by: Ben Sherman --- .../nextflow/processor/TaskProcessor.groovy | 8 ++++---- tests/resume-retried-task.nf | 20 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index d590ef143f..506c04ce87 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -1036,7 +1036,7 @@ class TaskProcessor { return true } - return fault == TERMINATE || fault == FINISH + return fault == ErrorStrategy.TERMINATE || fault == ErrorStrategy.FINISH } /** @@ -1050,7 +1050,7 @@ class TaskProcessor { final synchronized Object resumeOrDie( TaskRun task, Throwable error, TraceRecord traceRecord = null) { log.debug "Handling unexpected condition for\n task: name=${safeTaskName(task)}; work-dir=${task?.workDirStr}\n error [${error?.class?.name}]: ${error?.getMessage()?:error}" - ErrorStrategy errorStrategy = TERMINATE + ErrorStrategy errorStrategy = ErrorStrategy.TERMINATE final List message = [] try { // -- do not recoverable error, just re-throw it @@ -1169,7 +1169,7 @@ class TaskProcessor { // retry is not allowed when the script cannot be compiled or similar errors if( error instanceof ProcessUnrecoverableException || error.cause instanceof ProcessUnrecoverableException ) { - return !action.soft ? action : TERMINATE + return !action.soft ? action : ErrorStrategy.TERMINATE } // IGNORE strategy -- just continue @@ -1200,7 +1200,7 @@ class TaskProcessor { return ErrorStrategy.RETRY } - return TERMINATE + return ErrorStrategy.TERMINATE } return action diff --git a/tests/resume-retried-task.nf b/tests/resume-retried-task.nf index d7ecb85998..b3320392c5 100644 --- a/tests/resume-retried-task.nf +++ b/tests/resume-retried-task.nf @@ -20,16 +20,18 @@ process foo { maxRetries 3 script: - if( task.attempt < 3 ) - """ - exit 1 - """ - else - """ - echo ciao - """ + if( task.attempt < 3 ) { + """ + exit 1 + """ + } + else { + """ + echo ciao + """ + } } workflow { - foo() + foo() } From 401c89b0235b644f3d5549f2badf19197524d668 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 15 Apr 2026 15:32:24 -0500 Subject: [PATCH 6/9] Fix failing tests Signed-off-by: Ben Sherman --- .../groovy/nextflow/processor/TaskProcessor.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 506c04ce87..a42a292779 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -822,8 +822,8 @@ class TaskProcessor { // if the task execution failed or was aborted, increment the // try count so that any subsequent successful execution // can be recovered - if( entry.trace.isFailed() ) task.failCount += 1 - if( entry.trace.isAborted() ) task.abortedCount += 1 + if( entry?.trace?.isFailed() ) task.failCount += 1 + if( entry?.trace?.isAborted() ) task.abortedCount += 1 } catch (Throwable t) { log.warn1("[${safeTaskName(task)}] Unable to resume cached task -- See log file for details", causedBy: t) @@ -1062,7 +1062,7 @@ class TaskProcessor { log.info "[$task.hashLog] NOTE: ${error.message} -- Execution is retried" else log.info "[$task.hashLog] NOTE: ${error.message} -- Cause: ${error.cause.message} -- Execution is retried" - task.failCount+=1 + task.failCount += 1 final taskCopy = task.makeCopy() session.getExecService().submit { try { @@ -1075,8 +1075,8 @@ class TaskProcessor { } } task.failed = true - task.errorAction = RETRY - return RETRY + task.errorAction = ErrorStrategy.RETRY + return ErrorStrategy.RETRY } final submitTimeout = error.cause instanceof ProcessSubmitTimeoutException From 907a9cba6c2552f5edaaf2476ccff43151a01a60 Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 16 Apr 2026 15:23:08 +0200 Subject: [PATCH 7/9] change abortedCount to previousFailOrAborted Signed-off-by: jorgee --- .../src/main/groovy/nextflow/processor/TaskProcessor.groovy | 6 +++--- .../src/main/groovy/nextflow/processor/TaskRun.groovy | 5 +++-- tests/checks/resume-retried-with-abort.nf/.checks | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index a42a292779..c0e0461c6f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -797,7 +797,7 @@ class TaskProcessor { @CompileStatic final protected void checkCachedOrLaunchTask( TaskRun task, HashCode hash, boolean shouldTryCache ) { - int tries = task.failCount + task.abortedCount + 1 + int tries = task.failCount + task.previousFailOrAbortedCount + 1 while( true ) { // increment the task hash based on the "try count" // in order to produce a deterministic sequence of @@ -822,8 +822,8 @@ class TaskProcessor { // if the task execution failed or was aborted, increment the // try count so that any subsequent successful execution // can be recovered - if( entry?.trace?.isFailed() ) task.failCount += 1 - if( entry?.trace?.isAborted() ) task.abortedCount += 1 + if( entry?.trace?.isFailed() || entry?.trace?.isAborted() ) + task.previousFailOrAbortedCount += 1 } catch (Throwable t) { log.warn1("[${safeTaskName(task)}] Unable to resume cached task -- See log file for details", causedBy: t) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index a9d98bb1a5..d1bd73ad89 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -326,9 +326,10 @@ class TaskRun implements Cloneable { volatile int failCount /** - * The number of times the execution of the task has been aborted + * The number of times the execution of the task has been failed or aborted in other pipeline runs. + * Required to recover number of tries for task hash calculation without altering the number of attempts and failures in the current pipeline run. */ - volatile int abortedCount + volatile int previousFailOrAbortedCount /** * The number of times the submit of the task has been retried diff --git a/tests/checks/resume-retried-with-abort.nf/.checks b/tests/checks/resume-retried-with-abort.nf/.checks index e2cf1ab126..4aea9df918 100644 --- a/tests/checks/resume-retried-with-abort.nf/.checks +++ b/tests/checks/resume-retried-with-abort.nf/.checks @@ -21,9 +21,10 @@ timeout --signal=INT 12 $NXF_RUN -resume | tee stdout || true # RESUME mode with completed and abort # echo '' -timeout --signal=INT 12 $NXF_RUN -resume | tee stdout || true +timeout --signal=INT 20 $NXF_RUN -resume | tee stdout || true [[ `< .nextflow.log grep -c 'Submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false +[[ `< .nextflow.log grep -c 'Re-submitted process > SMALL_SLEEP_RETRY'` == 1 ]] || false # # RESUME mode with cached and finish From 146059ad4521235a17aa302c4f9693b2393ab2af Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 16 Apr 2026 09:51:26 -0500 Subject: [PATCH 8/9] rename `previousFailOrAbortedCount` -> `previousTryCount` Signed-off-by: Ben Sherman --- .../main/groovy/nextflow/processor/TaskProcessor.groovy | 4 ++-- .../src/main/groovy/nextflow/processor/TaskRun.groovy | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index c0e0461c6f..f4787e7b17 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -797,7 +797,7 @@ class TaskProcessor { @CompileStatic final protected void checkCachedOrLaunchTask( TaskRun task, HashCode hash, boolean shouldTryCache ) { - int tries = task.failCount + task.previousFailOrAbortedCount + 1 + int tries = task.previousTryCount + task.failCount + 1 while( true ) { // increment the task hash based on the "try count" // in order to produce a deterministic sequence of @@ -823,7 +823,7 @@ class TaskProcessor { // try count so that any subsequent successful execution // can be recovered if( entry?.trace?.isFailed() || entry?.trace?.isAborted() ) - task.previousFailOrAbortedCount += 1 + task.previousTryCount += 1 } catch (Throwable t) { log.warn1("[${safeTaskName(task)}] Unable to resume cached task -- See log file for details", causedBy: t) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index d1bd73ad89..65e53de852 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -326,10 +326,11 @@ class TaskRun implements Cloneable { volatile int failCount /** - * The number of times the execution of the task has been failed or aborted in other pipeline runs. - * Required to recover number of tries for task hash calculation without altering the number of attempts and failures in the current pipeline run. + * The number of unsuccessful attempts in the previous run. Used to + * recover the hash of a successful execution if it occurred after + * one or more failed attempts. */ - volatile int previousFailOrAbortedCount + volatile int previousTryCount /** * The number of times the submit of the task has been retried From 9e0f93ffab2ab69991a87a3d9eb90163d271e109 Mon Sep 17 00:00:00 2001 From: jorgee Date: Mon, 20 Apr 2026 14:39:31 +0200 Subject: [PATCH 9/9] Add unit test Signed-off-by: jorgee --- .../processor/TaskProcessorTest.groovy | 116 ++++++++++++++++++ .../nextflow/trace/TraceRecordTest.groovy | 34 +++++ 2 files changed, 150 insertions(+) diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy index a4f6c7ef2d..fc1c14b8dd 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy @@ -24,6 +24,7 @@ import java.util.concurrent.ExecutorService import com.google.common.hash.HashCode import groovyx.gpars.agent.Agent import nextflow.Session +import nextflow.cache.CacheDB import nextflow.exception.IllegalArityException import nextflow.exception.ProcessException import nextflow.exception.ProcessUnrecoverableException @@ -31,6 +32,8 @@ import nextflow.executor.Executor import nextflow.executor.NopeExecutor import nextflow.file.FileHolder import nextflow.file.FilePorter +import nextflow.trace.TraceRecord +import nextflow.util.HashBuilder import nextflow.script.BaseScript import nextflow.script.BodyDef import nextflow.script.ProcessConfig @@ -53,6 +56,13 @@ class TaskProcessorTest extends Specification { return new DummyProcessor(name, session, Mock(BaseScript), new ProcessConfig([:])) } + private static TraceRecord traceRecord(String status, String workdir) { + def rec = new TraceRecord() + rec.put('status', status) + rec.put('workdir', workdir) + return rec + } + static class DummyProcessor extends TaskProcessor { DummyProcessor(String name, Session session, BaseScript script, ProcessConfig config) { @@ -618,4 +628,110 @@ class TaskProcessorTest extends Specification { 0 * collector.collect(task) 1 * exec.submit(task) } + + def 'should increment previousTryCount when cache entry is failed'() { + given: + def baseDir = Files.createTempDirectory('test-cached') + def resumeDir = Files.createDirectory(baseDir.resolve('resume-dir')) + def cache = Mock(CacheDB) + def session = Mock(Session) { getCache() >> cache } + def executor = Mock(Executor) { getWorkDir() >> baseDir } + def processor = new TaskProcessor(session: session, executor: executor, name: 'foo') + and: + def task = new TaskRun(config: new TaskConfig(), name: 'foo', processor: processor) + + when: + processor.checkCachedOrLaunchTask(task, HashCode.fromString('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'), true) + + then: + // first lookup: failed entry with an existing resumeDir → forces next hash + 1 * cache.getTaskEntry(_, processor) >> new TaskEntry(trace: traceRecord('FAILED', resumeDir.toString())) + // second lookup: no entry → task is submitted + 1 * cache.getTaskEntry(_, processor) >> null + 1 * executor.submit(task) + and: + task.previousTryCount == 1 + + cleanup: + baseDir.toFile().deleteDir() + } + + def 'should increment previousTryCount when cache entry is aborted'() { + given: + def baseDir = Files.createTempDirectory('test-cached') + def resumeDir = Files.createDirectory(baseDir.resolve('resume-dir')) + def cache = Mock(CacheDB) + def session = Mock(Session) { getCache() >> cache } + def executor = Mock(Executor) { getWorkDir() >> baseDir } + def processor = new TaskProcessor(session: session, executor: executor, name: 'foo') + and: + def task = new TaskRun(config: new TaskConfig(), name: 'foo', processor: processor) + + when: + processor.checkCachedOrLaunchTask(task, HashCode.fromString('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'), true) + + then: + 1 * cache.getTaskEntry(_, processor) >> new TaskEntry(trace: traceRecord('ABORTED', resumeDir.toString())) + 1 * cache.getTaskEntry(_, processor) >> null + 1 * executor.submit(task) + and: + task.previousTryCount == 1 + + cleanup: + baseDir.toFile().deleteDir() + } + + def 'should not increment previousTryCount when no cache entry exists'() { + given: + def baseDir = Files.createTempDirectory('test-cached') + def cache = Mock(CacheDB) + def session = Mock(Session) { getCache() >> cache } + def executor = Mock(Executor) { getWorkDir() >> baseDir } + def processor = new TaskProcessor(session: session, executor: executor, name: 'foo') + and: + def task = new TaskRun(config: new TaskConfig(), name: 'foo', processor: processor) + + when: + processor.checkCachedOrLaunchTask(task, HashCode.fromString('cccccccccccccccccccccccccccccccc'), false) + + then: + 1 * cache.getTaskEntry(_, processor) >> null + 1 * executor.submit(task) + and: + task.previousTryCount == 0 + + cleanup: + baseDir.toFile().deleteDir() + } + + def 'should seed initial tries from previousTryCount and failCount'() { + given: + def baseDir = Files.createTempDirectory('test-cached') + def cache = Mock(CacheDB) + def session = Mock(Session) { getCache() >> cache } + def executor = Mock(Executor) { getWorkDir() >> baseDir } + def processor = new TaskProcessor(session: session, executor: executor, name: 'foo') + and: + def task = new TaskRun(config: new TaskConfig(), name: 'foo', processor: processor) + task.previousTryCount = 2 + task.failCount = 1 + def inputHash = HashCode.fromString('dddddddddddddddddddddddddddddddd') + + when: + processor.checkCachedOrLaunchTask(task, inputHash, false) + + then: + // no cached entry exists so the task is submitted immediately with the seeded hash + 1 * cache.getTaskEntry(_, processor) >> null + 1 * executor.submit(task) + and: + // the seeded hash corresponds to tries = previousTryCount + failCount + 1 = 4 + task.hash == HashBuilder.defaultHasher().putBytes(inputHash.asBytes()).putInt(4).hash() + and: + // no failed/aborted entry encountered, so counter is unchanged + task.previousTryCount == 2 + + cleanup: + baseDir.toFile().deleteDir() + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy b/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy index deabb2253d..d6d4fc8273 100644 --- a/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/trace/TraceRecordTest.groovy @@ -340,6 +340,40 @@ class TraceRecordTest extends Specification { 'COMPLETED' | true } + @Unroll + def 'should validate aborted status' () { + given: + def rec = new TraceRecord([status:STATUS]) + + expect: + rec.isAborted() == EXPECTED + + where: + STATUS | EXPECTED + null | false + 'NEW' | false + 'COMPLETED' | false + 'FAILED' | false + 'ABORTED' | true + } + + @Unroll + def 'should validate failed status' () { + given: + def rec = new TraceRecord([status:STATUS]) + + expect: + rec.isFailed() == EXPECTED + + where: + STATUS | EXPECTED + null | false + 'NEW' | false + 'COMPLETED' | false + 'ABORTED' | false + 'FAILED' | true + } + def 'should throw file not found exception' () { given: def rec = new TraceRecord([:])