Skip to content

Commit d963cc3

Browse files
authored
Use scan instead of copy in parallel mode (#13)
1 parent 2f57786 commit d963cc3

9 files changed

Lines changed: 27 additions & 25 deletions

File tree

.github/workflows/golangci-lint.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@ jobs:
2121
steps:
2222
- uses: actions/setup-go@v3
2323
with:
24-
go-version: 1.21.x
24+
go-version: 1.22.x
2525
- uses: actions/checkout@v2
2626
- name: golangci-lint
27-
uses: golangci/golangci-lint-action@v3.7.0
27+
uses: golangci/golangci-lint-action@v4.0.0
2828
with:
2929
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
30-
version: v1.55.2
30+
version: v1.56.2
3131

3232
# Optional: working directory, useful for monorepos
3333
# working-directory: somedir

.github/workflows/gorelease.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ concurrency:
99
cancel-in-progress: true
1010

1111
env:
12-
GO_VERSION: 1.21.x
12+
GO_VERSION: 1.22.x
1313
jobs:
1414
gorelease:
1515
runs-on: ubuntu-latest

.github/workflows/release-assets.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ on:
88
- created
99
env:
1010
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11-
GO_VERSION: '1.22.0-rc.1'
11+
GO_VERSION: 1.22.x
1212
LINUX_AMD64_BUILD_OPTIONS: '-tags cgo_zstd'
1313
GOAMD64: v3
1414
jobs:
1515
build:
1616
name: Upload Release Assets
17-
runs-on: ubuntu-20.04
17+
runs-on: ubuntu-latest
1818
steps:
1919
- name: Install Go stable
2020
if: env.GO_VERSION != 'tip'

.github/workflows/test-unit.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@ concurrency:
1515
env:
1616
GO111MODULE: "on"
1717
RUN_BASE_COVERAGE: "on" # Runs test for PR base in case base test coverage is missing.
18-
COV_GO_VERSION: 1.21.x # Version of Go to collect coverage
18+
COV_GO_VERSION: 1.22.x # Version of Go to collect coverage
1919
TARGET_DELTA_COV: 90 # Target coverage of changed lines, in percents
2020
jobs:
2121
test:
2222
strategy:
2323
matrix:
24-
go-version: [ 1.19.x, 1.20.x, 1.21.x ]
24+
go-version: [ 1.20.x, 1.21.x, 1.22.x ]
2525
runs-on: ubuntu-latest
2626
steps:
2727
- name: Install Go stable

.golangci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ linters-settings:
1919
funlen:
2020
lines: 80
2121
cyclop:
22-
max-complexity: 15
22+
max-complexity: 16
2323

2424
linters:
2525
enable-all: true

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#GOLANGCI_LINT_VERSION := "v1.55.2" # Optional configuration to pinpoint golangci-lint version.
1+
#GOLANGCI_LINT_VERSION := "v1.56.2" # Optional configuration to pinpoint golangci-lint version.
22

33
# The head of Makefile determines location of dev-go to include standard targets.
44
GO ?= go

cmd/catp/catp/app.go

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ func (r *runner) cat(filename string) (err error) {
279279
})
280280
}
281281

282-
if len(r.pass) > 0 || len(r.skip) > 0 {
282+
if len(r.pass) > 0 || len(r.skip) > 0 || r.parallel > 1 {
283283
r.scanFile(rd, out)
284284
} else {
285285
r.readFile(rd, out)
@@ -364,6 +364,8 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo
364364
skip stringFlags
365365
)
366366

367+
r := &runner{}
368+
367369
flag.Var(&pass, "pass", "filter matching, may contain multiple AND patterns separated by ^,\n"+
368370
"if filter matches, line is passed to the output (unless filtered out by -skip)\n"+
369371
"each -pass value is added with OR logic,\n"+
@@ -374,20 +376,21 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo
374376
"each -skip value is added with OR logic,\n"+
375377
"for example, you can use \"-skip quux^baz -skip fooO\" to skip lines that have (quux AND baz) OR fooO")
376378

377-
parallel := flag.Int("parallel", 1, "number of parallel readers if multiple files are provided\n"+
379+
flag.IntVar(&r.parallel, "parallel", 1, "number of parallel readers if multiple files are provided\n"+
378380
"lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file)\n"+
379381
"use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU)")
380382

381383
cpuProfile := flag.String("dbg-cpu-prof", "", "write first 10 seconds of CPU profile to file")
382384
memProfile := flag.String("dbg-mem-prof", "", "write heap profile to file after 10 seconds")
383385
output := flag.String("output", "", "output to file instead of STDOUT")
384-
outDir := flag.String("out-dir", "", "output to directory instead of STDOUT\n"+
385-
"files will be written to out dir with original base names\n"+
386-
"disables output flag")
387386
noProgress := flag.Bool("no-progress", false, "disable progress printing")
388387
progressJSON := flag.String("progress-json", "", "write current progress to a file")
389388
ver := flag.Bool("version", false, "print version and exit")
390389

390+
flag.StringVar(&r.outDir, "out-dir", "", "output to directory instead of STDOUT\n"+
391+
"files will be written to out dir with original base names\n"+
392+
"disables output flag")
393+
391394
flag.Usage = func() {
392395
fmt.Println("catp", version.Module("github.com/bool64/progress").Version+",",
393396
version.Info().GoVersion, strings.Join(versionExtra, " "))
@@ -420,12 +423,7 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo
420423
defer pprof.StopCPUProfile()
421424
}
422425

423-
r := &runner{}
424-
425-
r.parallel = *parallel
426-
r.outDir = *outDir
427-
428-
if *output != "" && *outDir == "" { //nolint:nestif
426+
if *output != "" && r.outDir == "" { //nolint:nestif
429427
out, err := os.Create(*output)
430428
if err != nil {
431429
return fmt.Errorf("failed to create output file %s: %w", *output, err)
@@ -503,7 +501,7 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo
503501
r.sizes[fn] = st.Size()
504502
}
505503

506-
if *parallel >= 2 {
504+
if r.parallel >= 2 {
507505
pr := r.pr
508506
pr.Start(func(t *progress.Task) {
509507
t.TotalBytes = func() int64 { return r.totalBytes }
@@ -513,7 +511,7 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo
513511
t.PrintOnStart = true
514512
})
515513

516-
sem := make(chan struct{}, *parallel)
514+
sem := make(chan struct{}, r.parallel)
517515
errs := make(chan error, 1)
518516

519517
for i := 0; i < flag.NArg(); i++ {

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
require (
66
github.com/DataDog/zstd v1.5.5
7-
github.com/bool64/dev v0.2.33
8-
github.com/klauspost/compress v1.17.4
7+
github.com/bool64/dev v0.2.34
8+
github.com/klauspost/compress v1.17.8
99
github.com/klauspost/pgzip v1.2.6
1010
)

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
22
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
33
github.com/bool64/dev v0.2.33 h1:ETAcSa8H9w4talcCdSQCCnLX7PMHmuxdLcDl6TpSDj4=
44
github.com/bool64/dev v0.2.33/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg=
5+
github.com/bool64/dev v0.2.34 h1:P9n315P8LdpxusnYQ0X7MP1CZXwBK5ae5RZrd+GdSZE=
6+
github.com/bool64/dev v0.2.34/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg=
57
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
68
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
9+
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
10+
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
711
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
812
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=

0 commit comments

Comments
 (0)