seqkit: add a global flag: --skip-file-check

shenwei356 · shenwei356 · commit 5b43ccc3aa33 · 2025-02-18T08:55:03.000+08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 - [SeqKit v2.10.0](https://github.com/shenwei356/seqkit/releases/tag/v2.10.0) - 2025-xx-xx
 [![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/seqkit/v2.10.0/total.svg)](https://github.com/shenwei356/seqkit/releases/tag/v2.10.0)
+    - `seqkit`:
+        - add a global flag `--skip-file-check`: skip input file checking when given a file list if you believe these files do exist.
+        It helps to reduce file checking time when given a huge number of sequence files.
     - `seqkit split2`:
         - fix prefix checking when paired-end files are given. [#512](https://github.com/shenwei356/seqkit/issues/512)
     - `seqkit stat`:
diff --git a/seqkit/cmd/amplicon.go b/seqkit/cmd/amplicon.go
@@ -121,7 +121,7 @@ Examples:
 		runtime.GOMAXPROCS(config.Threads)
 		bwt.CheckEndSymbol = false
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/bam.go b/seqkit/cmd/bam.go
@@ -501,7 +501,7 @@ var bamCmd = &cobra.Command{
 		outFile := config.OutFile
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		mapQual := getFlagInt(cmd, "map-qual")
 		field := getFlagString(cmd, "field")
diff --git a/seqkit/cmd/common.go b/seqkit/cmd/common.go
@@ -96,7 +96,7 @@ Note:
 			checkError(fmt.Errorf("flag -s (--by-seq) needed when using -e (--check-embedded-seqs)"))
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		if len(files) < 2 {
 			checkError(errors.New("at least 2 files needed"))
diff --git a/seqkit/cmd/concat.go b/seqkit/cmd/concat.go
@@ -70,7 +70,7 @@ Attention:
 		fill := fillBase != ""
 		separator := getFlagString(cmd, "separator")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		if len(files) < 2 {
 			checkError(errors.New("at least 2 files needed"))
diff --git a/seqkit/cmd/convert.go b/seqkit/cmd/convert.go
@@ -65,7 +65,7 @@ var convertCmd = &cobra.Command{
 		seq.NMostCommonThreshold = getFlagPositiveInt(cmd, "thresh-B-in-n-most-common")
 		threshIllumina1p5Frac := getFlagFloat64(cmd, "thresh-illumina1.5-frac")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/dup.go b/seqkit/cmd/dup.go
@@ -54,7 +54,7 @@ You may need "seqkit rename" to make the the sequence IDs unique.
 
 		times := getFlagPositiveInt(cmd, "times")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/fa2fq.go b/seqkit/cmd/fa2fq.go
@@ -62,7 +62,7 @@ Attention:
 		}
 		onlyPositiveStrand := getFlagBool(cmd, "only-positive-strand")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		records, err := fastx.GetSeqsMap(fileFasta, seq.Unlimit, config.Threads, 10, "")
 		checkError(err)
diff --git a/seqkit/cmd/fish.go b/seqkit/cmd/fish.go
@@ -162,7 +162,7 @@ Attention:
 			seq.ValidateSeq = true
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		var alns []*AlignedSeq
 		if len(files) == 0 {
diff --git a/seqkit/cmd/fq2fa.go b/seqkit/cmd/fq2fa.go
@@ -49,7 +49,7 @@ var fq2faCmd = &cobra.Command{
 		seq.ValidateSeq = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/fx2tab.go b/seqkit/cmd/fx2tab.go
@@ -60,7 +60,7 @@ Attention:
 		seq.ValidateSeq = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		onlyID := getFlagBool(cmd, "only-id")
 		printLength := getFlagBool(cmd, "length")
diff --git a/seqkit/cmd/grep.go b/seqkit/cmd/grep.go
@@ -98,7 +98,7 @@ Examples:
 
 		bwt.CheckEndSymbol = false
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		justCount := getFlagBool(cmd, "count")
 		pattern := getFlagStringSlice(cmd, "pattern")
diff --git a/seqkit/cmd/head-genome.go b/seqkit/cmd/head-genome.go
@@ -70,7 +70,7 @@ Attention:
 
 		minWords := getFlagPositiveInt(cmd, "mini-common-words")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/head.go b/seqkit/cmd/head.go
@@ -54,7 +54,7 @@ For returning the last N records, use:
 
 		number := getFlagPositiveInt(cmd, "number")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/helper.go b/seqkit/cmd/helper.go
@@ -245,6 +245,7 @@ type Config struct {
 	AlphabetGuessSeqLength int
 	ValidateSeqLength      int
 	CompressionLevel       int
+	SkipFileCheck          bool
 }
 
 func getConfigs(cmd *cobra.Command) Config {
@@ -294,6 +295,7 @@ func getConfigs(cmd *cobra.Command) Config {
 		Quiet:                  getFlagBool(cmd, "quiet"),
 		AlphabetGuessSeqLength: getFlagAlphabetGuessSeqLength(cmd, "alphabet-guess-seq-length"),
 		CompressionLevel:       level,
+		SkipFileCheck:          getFlagBool(cmd, "skip-file-check"),
 	}
 
 }
diff --git a/seqkit/cmd/locate.go b/seqkit/cmd/locate.go
@@ -77,7 +77,7 @@ Attention:
 
 		bwt.CheckEndSymbol = false
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		pattern := getFlagStringSlice(cmd, "pattern")
 		patternFile := getFlagString(cmd, "pattern-file")
diff --git a/seqkit/cmd/merge-slides.go b/seqkit/cmd/merge-slides.go
@@ -94,7 +94,7 @@ Output (BED3 format):
 		reQuery, err := regexp.Compile(reQueryStr)
 		checkError(err)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		if !config.Quiet {
 			if len(files) == 1 && isStdin(files[0]) {
diff --git a/seqkit/cmd/mutate.go b/seqkit/cmd/mutate.go
@@ -70,7 +70,7 @@ Examples:
 		runtime.GOMAXPROCS(config.Threads)
 		quiet := config.Quiet
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 		var err error
 
 		mPoints := []_mutatePoint{}
diff --git a/seqkit/cmd/range.go b/seqkit/cmd/range.go
@@ -90,7 +90,7 @@ Examples:
 		if start > 0 && end < 0 && end != -1 {
 			checkError(fmt.Errorf("not supported range: %d:%d, the end needs to be -1 when start > 0 and end < 0", start, end))
 		}
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/rename.go b/seqkit/cmd/rename.go
@@ -72,7 +72,7 @@ Example:
 		seq.ValidateSeq = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		byName := getFlagBool(cmd, "by-name")
 		mOutputs := getFlagBool(cmd, "multiple-outfiles")
diff --git a/seqkit/cmd/replace.go b/seqkit/cmd/replace.go
@@ -324,7 +324,7 @@ Filtering records to edit:
 
 		// -------------------
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/restart.go b/seqkit/cmd/restart.go
@@ -66,7 +66,7 @@ Examples
 		seq.ValidateSeq = false
 		runtime.GOMAXPROCS(config.BufferSize)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		newstart := getFlagInt(cmd, "new-start")
 		if newstart == 0 {
diff --git a/seqkit/cmd/rmdup.go b/seqkit/cmd/rmdup.go
@@ -81,7 +81,7 @@ Attention:
 			checkError(fmt.Errorf("flag -s (--by-seq) needed when using -P (--only-positive-strand)"))
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/root.go b/seqkit/cmd/root.go
@@ -133,6 +133,7 @@ func init() {
 	RootCmd.PersistentFlags().BoolP("quiet", "", false, "be quiet and do not show extra information")
 	RootCmd.PersistentFlags().IntP("alphabet-guess-seq-length", "", 10000, "length of sequence prefix of the first FASTA record based on which seqkit guesses the sequence type (0 for whole seq)")
 	RootCmd.PersistentFlags().StringP("infile-list", "X", "", "file of input files list (one file per line), if given, they are appended to files from cli arguments")
+	RootCmd.PersistentFlags().BoolP("skip-file-check", "", false, `skip input file checking when given a file list if you believe these files do exist`)
 	RootCmd.PersistentFlags().IntP("compress-level", "", -1, `compression level for gzip, zstd, xz and bzip2. type "seqkit -h" for the range and default value for each format`)
 
 	RootCmd.CompletionOptions.DisableDefaultCmd = true
diff --git a/seqkit/cmd/sample.go b/seqkit/cmd/sample.go
@@ -60,7 +60,7 @@ Attention:
 		seq.ValidateSeq = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		seed := getFlagInt64(cmd, "rand-seed")
 		twoPass := getFlagBool(cmd, "two-pass")
diff --git a/seqkit/cmd/sana.go b/seqkit/cmd/sana.go
@@ -87,7 +87,7 @@ Sana currently supports this FASTQ dialect:
 		allowGaps := getFlagBool(cmd, "allow-gaps")
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)
diff --git a/seqkit/cmd/seq.go b/seqkit/cmd/seq.go
@@ -131,7 +131,7 @@ var seqCmd = &cobra.Command{
 			checkError(fmt.Errorf("could not give both flags -l (--lower-case) and -u (--upper-case)"))
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		var seqCol *SeqColorizer
 		if color {
diff --git a/seqkit/cmd/shuffle.go b/seqkit/cmd/shuffle.go
@@ -71,7 +71,7 @@ Attention:
 		fai.MapWholeFile = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		seed := getFlagInt64(cmd, "rand-seed")
 		twoPass := getFlagBool(cmd, "two-pass")
diff --git a/seqkit/cmd/sliding.go b/seqkit/cmd/sliding.go
@@ -50,7 +50,7 @@ var slidingCmd = &cobra.Command{
 		seq.ValidateSeq = false
 		runtime.GOMAXPROCS(config.BufferSize)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		greedy := getFlagBool(cmd, "greedy")
 		circular := getFlagBool(cmd, "circular-genome") || getFlagBool(cmd, "circular")
diff --git a/seqkit/cmd/sort.go b/seqkit/cmd/sort.go
@@ -74,7 +74,7 @@ Attention:
 		fai.MapWholeFile = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		inNaturalOrder := getFlagBool(cmd, "natural-order")
 		bySeq := getFlagBool(cmd, "by-seq")
diff --git a/seqkit/cmd/split.go b/seqkit/cmd/split.go
@@ -87,7 +87,7 @@ Examples:
 		fai.MapWholeFile = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		if len(files) > 1 {
 			checkError(fmt.Errorf("no more than one file should be given"))
diff --git a/seqkit/cmd/split2.go b/seqkit/cmd/split2.go
@@ -89,7 +89,7 @@ If you want to cut a sequence into multiple segments.
 		fai.MapWholeFile = false
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		if len(files) > 1 {
 			checkError(fmt.Errorf("no more than one file should be given"))
diff --git a/seqkit/cmd/stat.go b/seqkit/cmd/stat.go
@@ -136,7 +136,7 @@ Tips:
 			}
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, !skipFileCheck, "infile-list", !skipFileCheck)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !(skipFileCheck || config.SkipFileCheck))
 
 		style := &stable.TableStyle{
 			Name: "plain",
diff --git a/seqkit/cmd/subseq.go b/seqkit/cmd/subseq.go
@@ -77,7 +77,7 @@ Examples:
 		Threads = config.Threads
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 		chrs := getFlagStringSlice(cmd, "chr")
 		chrs2 := make([]string, len(chrs))
 		for i, chr := range chrs {
diff --git a/seqkit/cmd/sum.go b/seqkit/cmd/sum.go
@@ -109,7 +109,7 @@ Examples:
 		rna2dna := getFlagBool(cmd, "rna2dna")
 		singleStrand := getFlagBool(cmd, "single-strand")
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		// process bar
 		var pbs *mpb.Progress
diff --git a/seqkit/cmd/tab2fx.go b/seqkit/cmd/tab2fx.go
@@ -46,7 +46,7 @@ var tab2faCmd = &cobra.Command{
 		outFile := config.OutFile
 		runtime.GOMAXPROCS(config.Threads)
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		commentPrefixes := getFlagStringSlice(cmd, "comment-line-prefix")
 		bufferSizeS := getFlagString(cmd, "buffer-size")
diff --git a/seqkit/cmd/translate.go b/seqkit/cmd/translate.go
@@ -162,7 +162,7 @@ Translate Tables/Genetic Codes:
 			return
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		var record *fastx.Record
 		var _seq *seq.Seq
diff --git a/seqkit/cmd/watch.go b/seqkit/cmd/watch.go
@@ -154,7 +154,7 @@ var watchCmd = &cobra.Command{
 			seq.ValidateSeq = true
 		}
 
-		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)
+		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)
 
 		outfh, err := xopen.Wopen(outFile)
 		checkError(err)

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ Note:`
`96`	`96`	`checkError(fmt.Errorf("flag -s (--by-seq) needed when using -e (--check-embedded-seqs)"))`
`97`	`97`	`}`
`98`	`98`
`99`		`- files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)`
	`99`	`+ files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)`
`100`	`100`
`101`	`101`	`if len(files) < 2 {`
`102`	`102`	`checkError(errors.New("at least 2 files needed"))`
Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,7 @@ Attention:`
`62`	`62`	`}`
`63`	`63`	`onlyPositiveStrand := getFlagBool(cmd, "only-positive-strand")`
`64`	`64`
`65`		`- files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)`
	`65`	`+ files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)`
`66`	`66`
`67`	`67`	`records, err := fastx.GetSeqsMap(fileFasta, seq.Unlimit, config.Threads, 10, "")`
`68`	`68`	`checkError(err)`
Original file line number	Diff line number	Diff line change
`@@ -162,7 +162,7 @@ Attention:`
`162`	`162`	`seq.ValidateSeq = true`
`163`	`163`	`}`
`164`	`164`
`165`		`- files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)`
	`165`	`+ files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", !config.SkipFileCheck)`
`166`	`166`
`167`	`167`	`var alns []*AlignedSeq`
`168`	`168`	`if len(files) == 0 {`