Merge branch 'master' into 2425-process-input-output-arity

Author: pditommaso

.github/workflows/build.yml

@@ -54,7 +54,7 @@ jobs:
         run: make assemble
 
       - name: Test
-        run: cat $HOME/.nextflow/scm; #make test
+        run: make test
 
       - name: Publish tests report
         uses: actions/upload-artifact@v3
@@ -72,7 +72,7 @@ jobs:
       fail-fast: false
       matrix:
         java_version: [11, 19]
-        test_mode: ["test_integration", "test_aws", "test_azure", "test_google"]
+        test_mode: ["test_integration", "test_aws", "test_azure", "test_google", "test_wave"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -86,6 +86,13 @@ jobs:
           echo "providers.github.auth='$NXF_GITHUB_ACCESS_TOKEN'" > "$HOME/.nextflow/scm"
         env:
           NXF_GITHUB_ACCESS_TOKEN: ${{ secrets.NXF_GITHUB_ACCESS_TOKEN }}
+      - name: Setup Java ${{ matrix.java_version }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{matrix.java_version}}
+          distribution: 'adopt'
+          architecture: x64
+          cache: gradle
       - name: Run tests
         run: |
           cat $HOME/.nextflow/scm

.gitignore

@@ -8,6 +8,7 @@
 .node-nextflow*
 .devcontainer
 .vscode/*
+*/*/bin/*
 **/build/**
 build/**
 modules/**/build/

Makefile

@@ -1,6 +1,5 @@
 #
-#  Copyright 2020-2022, Seqera Labs
-#  Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+#  Copyright 2013-2023, Seqera Labs
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

NOTICE

@@ -1,6 +1,6 @@
 NEXTFLOW
 
-Copyright 2020-2022, Seqera Labs
+Copyright 2020-2023, Seqera Labs
 Copyright 2013-2019, Centre for Genomic Regulation (CRG)
 
 
@@ -141,5 +141,5 @@ This software includes source code and libraries developed by:
     Copyright 2008, http://www.snakeyaml.org
     Licensed under Apache License, Version 2.0
 
-Nextflow is a registered trademark of Centre for Genomic Regulation, Spain (CRG).
+Nextflow is a registered trademark of Seqera Labs, Spain.
 See the trademark policy for details https://github.com/nextflow-io/trademark

README.md

@@ -16,14 +16,15 @@ Quick overview
 Nextflow is a bioinformatics workflow manager that enables the development of portable and reproducible workflows.
 It supports deploying workflows on a variety of execution platforms including local, HPC schedulers, AWS Batch,
 Google Cloud Life Sciences, and Kubernetes. Additionally, it provides support for manage your workflow dependencies
-through built-in support for Conda, Docker, Podmand, Singularity, and Modules.
+through built-in support for Conda, Spack, Docker, Podman, Singularity, Modules, and more.
 
 ## Contents
 - [Rationale](#rationale)
 - [Quick start](#quick-start)
 - [Documentation](#documentation)
 - [Tool Management](#tool-management)
   - [Conda environments](#conda-environments)
+  - [Spack environments](#spack-environments)
   - [Docker, Podman and Singularity](#containers)
   - [Environment Modules](#environment-modules)
 - [HPC Schedulers](#hpc-schedulers)
@@ -165,6 +166,12 @@ Conda environments
 [Conda environments](https://www.nextflow.io/docs/latest/conda.html) provide another option for managing software packages in your workflow. 
 
 
+Spack environments
+------------------
+
+Spack environments provide an option to build software packages from source using Spack, a popular package manager within the HPC community.
+
+
 Environment Modules
 -------
 

VERSION

@@ -1 +1 @@
-23.01.0-edge
+23.03.0-edge

build.gradle

@@ -1,6 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
- * Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -100,8 +99,8 @@ allprojects {
 
         // Documentation required libraries
         groovyDoc 'org.fusesource.jansi:jansi:1.11'
-        groovyDoc "org.codehaus.groovy:groovy-groovydoc:3.0.14"
-        groovyDoc "org.codehaus.groovy:groovy-ant:3.0.14"
+        groovyDoc "org.codehaus.groovy:groovy-groovydoc:3.0.16"
+        groovyDoc "org.codehaus.groovy:groovy-ant:3.0.16"
     }
 
     test {

buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy

@@ -1,6 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
- * Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

changelog.txt

@@ -1,8 +1,128 @@
 NEXTFLOW CHANGE-LOG
 ===================
+23.03.0-edge - 19 Mar 2023 
+- Add Azure batch pool virtualNetwork option (#3723) [e3917b8e]
+- Add basePackages option to Wave config [7e827810]
+- Add error message for missing container image with Google Batch (#3747) [6419e68f]
+- Add notifyFilePublish optional source argument (#3737) [a65a9ff7]
+- Add retry strategy on Bash wrapper write logic [1e38274a]
+- Add support for AWS S3 Glacier Retrieval Tier (#3680) [fab6bd5e]
+- Add sync invocation on job completion (#3678) [e29c4e4b]
+- Add workflowId to Wave request [025ff9d0]
+- Add Clone result list objects before output binding [38762080]
+- Add plugins option to kuberun command (#3766) [6515294f]
+- Fix Google Batch logging exception [d7e38e9e]
+- Fix Java Matcher serialization [6aecc4ed]
+- Fix Logger failure type detection [f1b89c1f]
+- Fix Orphaned processes when using Singularity [7236f830]
+- Fix groupTuple with mismatching tuple cardinality [1d32e2c2]
+- Fix special chars escaping in container env [b93634e9]
+- Fix commit revision type for Azure DevOps repos (#3721) [fb71767e]
+- Improve DAG API (#3703) [21afb358]
+- Improve Wave docs [7299dac9]
+- Improve `fair` directive docs (#3677) [7ca7520f]
+- Improve transpose and groupTuple examples in the docs (#3685) [7870dfc8]
+- Increase pwd obfuscation min length [ba23d036]
+- Make GroupKey cloneable (#3738) [0760b0fd]
+- Prepare Nextflow for the CWS Plugin (#3727) [fd8bf709]
+- Remove DNAnexus page from docs (#3695) [fa58c18c]
+- Remove outdated FAQ page from docs (#3683) [e59c7fbb]
+- Remove unused classes [9fa8d75b]
+- Update Azure VM sizes (#3751) [1d06e9a6]
+- Enable Fusion on Google Batch tests [2f1b1094]
+- Bump groovy 3.0.16 [d3ff5dcb]
+- Bump nf-wave@0.8.0 [aa1b065d]
+- Bump nf-google@1.7.1 [2aaaf2a7]
+- Bump nf-azure@0.16.0 [f358b008]
+- Bump nf-amazon@1.16.0 [2395187d]
+
+23.02.1-edge - 26 Feb 2023
+- Add Fusion logs and tags config [09192078]
+- Add support for NXF_DEBUG to K8s task handler [828583dc]
+- Add support for Mamba HTTP based lock files [7fe88c84]
+- Add container name validation [c3b6d9ab]
+- Add Fusion for Google storage docs [34fcfe06]
+- Improve K8s auth token discovery [44f8cc07]
+- Bump nf-wave@0.7.2 [1c697ab5]
+- Bump amazoncorretto:17.0.6 [a33a6f47]
+
+23.02.0-edge - 21 Feb 2023
+- Add Fusion support for Google Batch (#3577) [d5fbab4f]
+- Add Header provider to Google Batch client [20979929]
+- Add Wave integration tests [c5faf971]
+- Add errorStrategy validation [954cd921]
+- Add Java install recommendations to docs (#3617) [7e9bee1e]
+- Add nextflow tags to AWS Batch job def [b465ac52] [BREAKING]
+- Add readOnlyInputs option for CharliecloudBuilder (#3590) [8ae001fc]
+- Add refresh token to Wave request [41510e87]
+- Add requirements for container images to run with Nextflow (#3586) [7af1cb66]
+- Add support for Fusion to Slurm and LSF executors (#3516) [fbc95e6b]
+- Add support for Fusion tags (#3609) [8385ec22] [BREAKING]
+- Add support for the Spack package manager (#3580) [a5df62f6]
+- Add the fcp tool option to file copy strategy (#3556) [2a224258]
+- Add Fusion integration test 
+- Add Wave + Fusion test with Google Batch [b541c061]
+- Add support for entryless Fusion configuration and Singularity support (#3584) [34a27733]
+- Improve Fusion config [a1d08463]
+- Ignore resource directives if -l option is specified in PBS executor (#3015) [28a67969]
+- Improve K8s securityContext support [3f76240b]
+- Improve AWS Batch error logging [8f4884c1]
+- Improve secrets docs [9a3b4214]
+- Improve Wave debug-task plugin command [cf3fd747]
+- Improve Wave logging [fced3761]
+- Increase Wave token duration to 30 mins [e1a712c2]
+- Fix -with-apptainer cli option (#3621) [8c9588fe]
+- Fix GH action tests step [bb890d9a]
+- Fix Spack failing test [868374d3]
+- Fix Tower doc links, fix typos (#3647) [1278e1a5]
+- Fix Wave cli messages [10cf414b]
+- Fix collectFile cache over remote storage [762ba19c]
+- Fix output env var tokenize [40d56735]
+- Fix plugin install version [59e9a91b]
+- Fix race condition on Wave requests [f57fae2c]
+- Fix serialization of S3 paths with spaces (#3565) [ce487624]
+- Fix string literal method names with parens (#3604) [8278078a]
+- Fix submit command in error message for grid executors that pipe wrapper script (#3548) [40ebd308]
+- Fix typos in the documentation [skip ci] (#3640) [eccc1130]
+- Fix unique fingerprint for Wave bundle resources [b59fa728]
+- Log exception when an unexpected  occurs (#3603) [9aeca99f]
+- Minor change on container resolution [23d47ded]
+- Refactor httpfs configuration [445218a8]
+- Remove deprecated buckets field [0a355ac3]
+- Remove leading/trailing special chars from tag [7d3cc707]
+- Remove unused `-with-k8s` CLI option (#3624) [85b6e7d3]
+- Update config.rst (#3655) [1555e188]
+- Update Fusion docs [12782eeb]
+- Bump Fusion 2.1 [f48ef5b3]
+- Bump Groovy 3.0.15 [7a3ebc7d]
+- Bump Micromamba 1.3.1 [aa2bf37e]
+- Bump Spock 2.3 [9f0dac31]
+- Bump nf-wave@0.7.1 [b797fac2]
+- Bump nf-tower@1.5.10 [dd2f69dd]
+- Bump nf-google@1.7.0 [8cb9fa35]
+- Bump nf-amazon@1.15.0 [7ccca430]
+
+22.10.7 - 18 Feb 2023
+- Fix plugin install version [895e4f34]
+- Fix unique fingerprint for wave bundle resources [7c55aa37]
+- Fix race condition on wave requests [7f76d19b]
+- Fix serialization of S3 paths with spaces (#3565) [935df949]
+- Improve aws batch error logging [1eddeea7]
+- Improve wave logging [94415914]
+- Increase Wave token duration to 30 mins [8d5fbefe]
+- Update fusion docs [b24ee138]
+- Bump nf-wave@0.5.4 [f466b266]
+- Bump nf-amazon@1.11.4 [d5f6c125]
+
+22.10.6 - 24 Jan 2023
+- backport: Add refresh token to wave request [47a7408b]
+- backport:Add tower endpoint to wave [fe24b422]
+- backport:Fix Wave layer invalid checksum due to not closed stream [20df6400]
+- Bump nf-wave@0.5.3 [c1ec759f]
+
 23.01.0-edge - 14 Jan 2023 
 - Add support for rclone as stageOutMode option [fde2ac27]
-- Add queueGlobalStatus config setting [ci fast] [f3339541]
+- Add queueGlobalStatus config setting [f3339541]
 - Add support for bearer token to GithubRepositoryProvider [c4d39384] [a2d2cbf60]
 - Add -with-fusion cli option [d22337cd]
 - Add support for Fusion container custom options [56e29d35]
@@ -13,15 +133,15 @@ NEXTFLOW CHANGE-LOG
 - Fix GitHub creds in CI build [4c4d3f81]
 - Fix FilePorter concurrency issue (#3511) [11ccfa26]
 - Fix watchPath hang shutdown execution [bac0cedf]
-- Fix "no such file" error message to "no such file or directory" [ci fast] (#3547) [c292d84e]
+- Fix "no such file" error message to "no such file or directory" (#3547) [c292d84e]
 - Improve local exec workdir validation [4a3a2720]
 - Improve container native executor configuration [03126371]
 - Improve AWS batch exit code reporting [d1bb2fe2]
 - Improve channel factories error message [0215b9fc]
 - Improve Google Batch executor [c0a25fc2]
 - Improve Fusion config [902e5b34] [52f4c5d5]
 - Improve redirection on Wave client [124cfb3e]
-- Remove unnecessary Fusion env var [ci fast] [dfa47556]
+- Remove unnecessary Fusion env var [dfa47556]
 - Remove deprecated method 'just' [d17b3c46]
 - Remove deprecated methods [75e65035]
 - Remove DSL1 deprecated code [5d290a47]

config/codenarc/codenarc.groovy

@@ -1,6 +1,5 @@
 /*
- * Copyright 2020-2022, Seqera Labs
- * Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+ * Copyright 2013-2023, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -349,4 +348,4 @@ ruleset {
     UnusedVariable 
 
 
-}
+}