Skip to content

Commit 641b6bf

Browse files
danlapiddanlapid
authored
Develop
* Added watcher tests * Added test reporting and test summary to ci * Replaced Uber rate limiter with Golang rate limiter * Added buffer fill support to windows and linux
1 parent f9023ad commit 641b6bf

29 files changed

Lines changed: 896 additions & 244 deletions

File tree

‎.github/workflows/coverage.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Coverage
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
branches:
8+
- main
9+
jobs:
10+
coverage:
11+
strategy:
12+
fail-fast: false
13+
matrix:
14+
os:
15+
- ubuntu-latest
16+
go:
17+
- "1.19"
18+
runs-on: ${{ matrix.os }}
19+
steps:
20+
- name: Checkout code
21+
uses: actions/checkout@v3
22+
with:
23+
fetch-depth: 1
24+
25+
- name: Install Go
26+
uses: actions/setup-go@v3
27+
with:
28+
go-version: ${{ matrix.go }}
29+
30+
- name: Calc coverage
31+
run: |
32+
go test -v -covermode=count -coverprofile=coverage.out -coverpkg ./pkg/... ./...
33+
- name: Convert coverage.out to coverage.lcov
34+
uses: jandelgado/gcov2lcov-action@master
35+
- name: Coveralls
36+
uses: coverallsapp/github-action@master
37+
with:
38+
github-token: ${{ secrets.GITHUB_TOKEN }}
39+
path-to-lcov: coverage.lcov

‎.github/workflows/test.yml‎

Lines changed: 19 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -30,36 +30,28 @@ jobs:
3030
go-version: ${{ matrix.go }}
3131

3232
- name: test
33+
if: runner.os != 'Windows'
3334
run: |
34-
go test -v -race ./...
35+
go install github.com/jstemmer/go-junit-report/v2@latest
36+
go test -v -race ./... 2>&1 | go-junit-report -set-exit-code -iocopy -out report.xml
3537
36-
coverage:
37-
strategy:
38-
fail-fast: false
39-
matrix:
40-
os:
41-
- ubuntu-latest
42-
go:
43-
- "1.19"
44-
runs-on: ${{ matrix.os }}
45-
steps:
46-
- name: Checkout code
47-
uses: actions/checkout@v3
48-
with:
49-
fetch-depth: 1
38+
- name: test-windows
39+
if: runner.os == 'Windows'
40+
run: |
41+
go install github.com/jstemmer/go-junit-report/v2@latest
42+
go test -v ./... 2>&1 | go-junit-report -set-exit-code -iocopy -out report.xml
5043
51-
- name: Install Go
52-
uses: actions/setup-go@v3
44+
- name: Test Report
45+
uses: dorny/test-reporter@v1
5346
with:
54-
go-version: ${{ matrix.go }}
47+
name: ${{ matrix.os }} Tests
48+
path: report.xml
49+
reporter: java-junit
50+
fail-on-error: "false"
51+
if: always()
5552

56-
- name: Calc coverage
57-
run: |
58-
go test -v -covermode=count -coverprofile=coverage.out -coverpkg ./pkg/... ./...
59-
- name: Convert coverage.out to coverage.lcov
60-
uses: jandelgado/gcov2lcov-action@master
61-
- name: Coveralls
62-
uses: coverallsapp/github-action@master
53+
- name: Test Summary
54+
uses: test-summary/action@v1
6355
with:
64-
github-token: ${{ secrets.GITHUB_TOKEN }}
65-
path-to-lcov: coverage.lcov
56+
paths: report.xml
57+
if: always()

‎cmd/receiver/main.go‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@ import (
66
"oneway-filesync/pkg/database"
77
"oneway-filesync/pkg/receiver"
88
"oneway-filesync/pkg/utils"
9-
"os"
10-
"os/signal"
11-
"syscall"
129

1310
"github.com/sirupsen/logrus"
1411
)
@@ -28,16 +25,9 @@ func main() {
2825
return
2926
}
3027

31-
if err = database.ConfigureDatabase(db); err != nil {
32-
logrus.Errorf("Failed setting up db with err %v", err)
33-
return
34-
}
35-
3628
ctx, cancel := context.WithCancel(context.Background()) // Create a cancelable context and pass it to all goroutines, allows us to gracefully shut down the program
3729
receiver.Receiver(ctx, db, conf)
3830

39-
done := make(chan os.Signal, 1)
40-
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
41-
<-done
31+
<-utils.CtrlC()
4232
cancel() // Gracefully shutdown and stop all goroutines
4333
}

‎cmd/sender/main.go‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@ import (
66
"oneway-filesync/pkg/database"
77
"oneway-filesync/pkg/sender"
88
"oneway-filesync/pkg/utils"
9-
"os"
10-
"os/signal"
11-
"syscall"
129

1310
"github.com/sirupsen/logrus"
1411
)
@@ -27,16 +24,9 @@ func main() {
2724
return
2825
}
2926

30-
if err = database.ConfigureDatabase(db); err != nil {
31-
logrus.Errorf("Failed setting up db with err %v", err)
32-
return
33-
}
34-
3527
ctx, cancel := context.WithCancel(context.Background()) // Create a cancelable context and pass it to all goroutines, allows us to gracefully shut down the program
3628
sender.Sender(ctx, db, conf)
3729

38-
done := make(chan os.Signal, 1)
39-
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
40-
<-done
30+
<-utils.CtrlC()
4131
cancel() // Gracefully shutdown and stop all goroutines
4232
}

‎cmd/sendfiles/main.go‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,6 @@ func main() {
1919
return
2020
}
2121

22-
if err = database.ConfigureDatabase(db); err != nil {
23-
fmt.Printf("Failed setting up db with err %v\n", err)
24-
return
25-
}
26-
2722
path := os.Args[1]
2823
err = filepath.Walk(path, func(filepath string, info os.FileInfo, e error) error {
2924
if !info.IsDir() {

‎cmd/watcher/main.go‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,31 @@ package main
22

33
import (
44
"context"
5+
"oneway-filesync/pkg/config"
56
"oneway-filesync/pkg/database"
67
"oneway-filesync/pkg/utils"
78
"oneway-filesync/pkg/watcher"
8-
"os"
9-
"os/signal"
10-
"syscall"
119

12-
"github.com/rjeczalik/notify"
1310
"github.com/sirupsen/logrus"
1411
)
1512

1613
func main() {
1714
utils.InitializeLogging("watcher.log")
18-
if len(os.Args) < 2 {
19-
logrus.Errorf("Usage: %s <dir_path>", os.Args[0])
15+
conf, err := config.GetConfig("config.toml")
16+
if err != nil {
17+
logrus.Errorf("Failed reading config with err %v", err)
2018
return
2119
}
22-
path := os.Args[1]
2320

2421
db, err := database.OpenDatabase("s_")
2522
if err != nil {
2623
logrus.Errorf("%v", err)
2724
return
2825
}
2926

30-
if err = database.ConfigureDatabase(db); err != nil {
31-
logrus.Errorf("Failed setting up db with err %v", err)
32-
return
33-
}
34-
35-
events := make(chan notify.EventInfo, 20)
3627
ctx, cancel := context.WithCancel(context.Background()) // Create a cancelable context and pass it to all goroutines, allows us to gracefully shut down the program
37-
watcher.CreateWatcher(ctx, db, path, events)
28+
watcher.Watcher(ctx, db, conf)
3829

39-
done := make(chan os.Signal, 1)
40-
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
41-
<-done
30+
<-utils.CtrlC()
4231
cancel() // Gracefully shutdown and stop all goroutines
4332
}

‎config.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ ChunkSize = 8192
55
ChunkFecRequired = 5
66
ChunkFecTotal = 10
77
OutDir = "./out"
8+
WatchDir = "./tmp"

‎go.mod‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,16 @@ go 1.19
55
require (
66
github.com/BurntSushi/toml v1.2.0
77
github.com/klauspost/reedsolomon v1.10.0
8-
github.com/rjeczalik/notify v0.9.2
8+
github.com/rjeczalik/notify v0.9.3-0.20210809113154-3472d85e95cd
99
github.com/sirupsen/logrus v1.9.0
1010
github.com/zhuangsirui/binpacker v2.0.0+incompatible
11-
go.uber.org/ratelimit v0.2.0
12-
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64
11+
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261
12+
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
1313
gorm.io/driver/sqlite v1.3.6
1414
gorm.io/gorm v1.23.8
1515
)
1616

1717
require (
18-
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
1918
github.com/jinzhu/inflection v1.0.0 // indirect
2019
github.com/jinzhu/now v1.1.5 // indirect
2120
github.com/klauspost/cpuid/v2 v2.0.14 // indirect

‎go.sum‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
22
github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
3-
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
4-
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
53
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
64
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
75
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -20,23 +18,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
2018
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
2119
github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
2220
github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
21+
github.com/rjeczalik/notify v0.9.3-0.20210809113154-3472d85e95cd h1:LHLg0gdpRUCvujg2Zol6e2Uknq5vHycLxqEzYwxt1vY=
22+
github.com/rjeczalik/notify v0.9.3-0.20210809113154-3472d85e95cd/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=
2323
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
2424
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
2525
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
26-
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
27-
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
2826
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
2927
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
3028
github.com/zhuangsirui/binpacker v2.0.0+incompatible h1:s2wDYWXT4IznT7NUFzn5gJHqjtWz/zIwUxdiFGNomdk=
3129
github.com/zhuangsirui/binpacker v2.0.0+incompatible/go.mod h1:TdE7uEZ8Q7sMzbCpk2Y+ksFB8yA5AErPz0meDB612rU=
32-
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
33-
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
34-
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
35-
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
3630
golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
3731
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
38-
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
39-
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
32+
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
33+
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
34+
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
35+
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
4036
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
4137
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
4238
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎pkg/bandwidthlimiter/bandwidthlimiter.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ import (
44
"context"
55
"oneway-filesync/pkg/structs"
66

7-
"go.uber.org/ratelimit"
7+
"github.com/sirupsen/logrus"
8+
"golang.org/x/time/rate"
89
)
910

1011
type bandwidthLimiterConfig struct {
11-
rl ratelimit.Limiter
12+
rl *rate.Limiter
1213
input chan *structs.Chunk
1314
output chan *structs.Chunk
1415
}
@@ -19,15 +20,18 @@ func worker(ctx context.Context, conf *bandwidthLimiterConfig) {
1920
case <-ctx.Done():
2021
return
2122
case buf := <-conf.input:
22-
conf.rl.Take()
23-
conf.output <- buf
23+
if err := conf.rl.WaitN(ctx, len(buf.Data)); err != nil {
24+
logrus.Error(err)
25+
} else {
26+
conf.output <- buf
27+
}
2428
}
2529
}
2630
}
2731

28-
func CreateBandwidthLimiter(ctx context.Context, chunks_per_sec int, input chan *structs.Chunk, output chan *structs.Chunk) {
32+
func CreateBandwidthLimiter(ctx context.Context, bandwidth int, chunksize int, input chan *structs.Chunk, output chan *structs.Chunk) {
2933
conf := bandwidthLimiterConfig{
30-
rl: ratelimit.New(chunks_per_sec),
34+
rl: rate.NewLimiter(rate.Limit(bandwidth), chunksize),
3135
input: input,
3236
output: output,
3337
}

0 commit comments

Comments
 (0)