Skip to content

Commit 9e2e7eb

Browse files
bentshermanclaude
andcommitted
Include module resources bundle in task hash for -resume caching
When a module uses local binaries via its `resources/` directory (the `nextflow.enable.moduleBinaries` feature flag), those files were not part of the task hash used for `-resume` caching. As a result, editing a module's `resources` script did not invalidate the cache and `-resume` returned a stale cached result instead of re-executing the task. Add the module resources bundle fingerprint to the task hash key list in `TaskHasher.compute()`, guarded by `session.enableModuleBinaries()` (matching the existing guard in `TaskProcessor.getBinDirs()`), so edits to module binaries correctly force task re-execution while unchanged resources still hit the cache. Note: this does not affect Wave setups, where the resources are baked into the container and already covered by the container fingerprint. Fixes #6128 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Ben Sherman <bentshermann@gmail.com>
1 parent 79594e7 commit 9e2e7eb

2 files changed

Lines changed: 48 additions & 0 deletions

File tree

modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,15 @@ class TaskHasher {
9696
keys.addAll(binEntries)
9797
}
9898

99+
// add the fingerprint of the module resources bundle (e.g. module binaries)
100+
// since these files are not staged individually like the project bin dir
101+
final moduleBundle = session.enableModuleBinaries() ? processor.getModuleBundle() : null
102+
if( moduleBundle && moduleBundle.hasEntries() ) {
103+
final fingerprint = moduleBundle.fingerprint()
104+
log.trace "Task: ${task.processor.name} > Adding module bundle fingerprint: ${fingerprint}"
105+
keys.add(fingerprint)
106+
}
107+
99108
// add environment modules (`module` directive)
100109
final modules = task.getConfig().getModule()
101110
if( modules ) {

modules/nextflow/src/test/groovy/nextflow/processor/TaskHasherTest.groovy

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import java.nio.file.Path
2020

2121
import nextflow.Session
2222
import nextflow.script.ProcessConfig
23+
import nextflow.script.bundle.ResourcesBundle
2324
import spock.lang.Specification
2425
/**
2526
*
@@ -94,6 +95,44 @@ class TaskHasherTest extends Specification {
9495
result.contains(Path.of('/some/path/bar.sh'))
9596
}
9697

98+
def 'should include module bundle fingerprint in the task hash' () {
99+
100+
given: 'two module bundles with the same fingerprint, then one with a different fingerprint'
101+
def bundleA1 = Mock(ResourcesBundle) { asBoolean() >> true; hasEntries() >> true; fingerprint() >> 'aaaaaaaa' }
102+
def bundleA2 = Mock(ResourcesBundle) { asBoolean() >> true; hasEntries() >> true; fingerprint() >> 'aaaaaaaa' }
103+
def bundleB = Mock(ResourcesBundle) { asBoolean() >> true; hasEntries() >> true; fingerprint() >> 'bbbbbbbb' }
104+
and: 'a task whose process resolves those bundles across successive hash computations'
105+
def session = Mock(Session) {
106+
getUniqueId() >> UUID.fromString('b69b6eeb-b332-4d2c-9957-c291b15f498c')
107+
getBinEntries() >> [:]
108+
enableModuleBinaries() >> true
109+
}
110+
def processor = Mock(TaskProcessor) {
111+
getName() >> 'hello'
112+
getSession() >> session
113+
getConfig() >> Mock(ProcessConfig)
114+
getModuleBundle() >>> [bundleA1, bundleA2, bundleB]
115+
}
116+
def task = Mock(TaskRun) {
117+
getSource() >> 'hello world'
118+
isContainerEnabled() >> false
119+
getConfig() >> Mock(TaskConfig)
120+
getProcessor() >> processor
121+
}
122+
def hasher = Spy(new TaskHasher(task))
123+
hasher.getTaskGlobalVars() >> [:]
124+
125+
when:
126+
def hashA1 = hasher.compute()
127+
def hashA2 = hasher.compute()
128+
def hashB = hasher.compute()
129+
130+
then: 'the same bundle fingerprint yields the same hash'
131+
hashA1 == hashA2
132+
and: 'a different bundle fingerprint invalidates the cache'
133+
hashA1 != hashB
134+
}
135+
97136
def 'should get task directive vars' () {
98137
given:
99138
def processor = Spy(TaskProcessor) {

0 commit comments

Comments
 (0)