Skip to content

Commit eddf16e

Browse files
danlapiddanlapid
authored
More tests
1 parent 84fb58c commit eddf16e

10 files changed

Lines changed: 335 additions & 40 deletions

File tree

‎pkg/fecdecoder/fecdecoder_test.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package fecdecoder
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"oneway-filesync/pkg/structs"
7+
"strings"
8+
"testing"
9+
"time"
10+
11+
"github.com/klauspost/reedsolomon"
12+
"github.com/sirupsen/logrus"
13+
)
14+
15+
func createChunks(t *testing.T, required int, total int) []*structs.Chunk {
16+
fec, err := reedsolomon.New(required, total-required)
17+
if err != nil {
18+
t.Fatal(err)
19+
}
20+
shares, err := fec.Split(make([]byte, 400))
21+
if err != nil {
22+
t.Fatal(err)
23+
}
24+
25+
// Encode the parity set
26+
err = fec.Encode(shares)
27+
if err != nil {
28+
t.Fatal(err)
29+
}
30+
chunks := make([]*structs.Chunk, total)
31+
for i, sharedata := range shares {
32+
chunks[i] = &structs.Chunk{
33+
ShareIndex: uint32(i),
34+
Data: sharedata,
35+
}
36+
}
37+
return chunks
38+
39+
}
40+
41+
func Test_worker(t *testing.T) {
42+
type args struct {
43+
required int
44+
total int
45+
input []*structs.Chunk
46+
}
47+
tests := []struct {
48+
name string
49+
args args
50+
wantErr bool
51+
expectedErr string
52+
}{
53+
{"test-works", args{2, 4, createChunks(t, 2, 4)}, false, ""},
54+
{"test-too-few-shards", args{4, 8, createChunks(t, 4, 8)[:3]}, true, "Error FEC decoding shares: too few shards given"},
55+
{"test-invalid-fec1", args{2, 1, make([]*structs.Chunk, 4)}, true, "Error creating fec object: cannot create Encoder with less than one data shard or less than zero parity shards"},
56+
{"test-invalid-fec2", args{0, 1, make([]*structs.Chunk, 4)}, true, "Error creating fec object: cannot create Encoder with less than one data shard or less than zero parity shards"},
57+
}
58+
for _, tt := range tests {
59+
t.Run(tt.name, func(t *testing.T) {
60+
var memLog bytes.Buffer
61+
logrus.SetOutput(&memLog)
62+
63+
input := make(chan []*structs.Chunk, 5)
64+
output := make(chan *structs.Chunk, 5)
65+
66+
input <- tt.args.input
67+
68+
conf := fecDecoderConfig{tt.args.required, tt.args.total, input, output}
69+
ctx, cancel := context.WithCancel(context.Background())
70+
go func() {
71+
time.Sleep(2 * time.Second)
72+
cancel()
73+
}()
74+
// ch <- tt.args.file
75+
worker(ctx, &conf)
76+
77+
if tt.wantErr {
78+
if !strings.Contains(memLog.String(), tt.expectedErr) {
79+
t.Fatalf("Expected not in log, '%v' not in '%v'", tt.expectedErr, memLog.String())
80+
}
81+
} else {
82+
<-output
83+
}
84+
85+
})
86+
}
87+
}

‎pkg/fecencoder/fecencoder.go‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,10 @@ import (
1010
)
1111

1212
type fecEncoderConfig struct {
13-
chunksize int
14-
required int
15-
total int
16-
input chan *structs.Chunk
17-
output chan *structs.Chunk
13+
required int
14+
total int
15+
input chan *structs.Chunk
16+
output chan *structs.Chunk
1817
}
1918

2019
// FEC routine:
@@ -71,13 +70,12 @@ func worker(ctx context.Context, conf *fecEncoderConfig) {
7170
}
7271
}
7372

74-
func CreateFecEncoder(ctx context.Context, chunksize int, required int, total int, input chan *structs.Chunk, output chan *structs.Chunk, workercount int) {
73+
func CreateFecEncoder(ctx context.Context, required int, total int, input chan *structs.Chunk, output chan *structs.Chunk, workercount int) {
7574
conf := fecEncoderConfig{
76-
chunksize: chunksize,
77-
required: required,
78-
total: total,
79-
input: input,
80-
output: output,
75+
required: required,
76+
total: total,
77+
input: input,
78+
output: output,
8179
}
8280
for i := 0; i < workercount; i++ {
8381
go worker(ctx, &conf)

‎pkg/fecencoder/fecencoder_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package fecencoder
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"oneway-filesync/pkg/structs"
7+
"strings"
8+
"testing"
9+
"time"
10+
11+
"github.com/sirupsen/logrus"
12+
)
13+
14+
func Test_worker(t *testing.T) {
15+
type args struct {
16+
required int
17+
total int
18+
input *structs.Chunk
19+
}
20+
tests := []struct {
21+
name string
22+
args args
23+
wantErr bool
24+
expectedErr string
25+
}{
26+
{"test-works", args{2, 4, &structs.Chunk{Data: make([]byte, 400)}}, false, ""},
27+
{"test-shortdata1", args{2, 4, &structs.Chunk{Data: make([]byte, 0)}}, true, "Error splitting chunk: not enough data to fill the number of requested shards"},
28+
{"test-invalid-fec1", args{2, 1, &structs.Chunk{}}, true, "Error creating fec object: cannot create Encoder with less than one data shard or less than zero parity shards"},
29+
{"test-invalid-fec2", args{0, 1, &structs.Chunk{}}, true, "Error creating fec object: cannot create Encoder with less than one data shard or less than zero parity shards"},
30+
}
31+
for _, tt := range tests {
32+
t.Run(tt.name, func(t *testing.T) {
33+
var memLog bytes.Buffer
34+
logrus.SetOutput(&memLog)
35+
36+
input := make(chan *structs.Chunk, 5)
37+
output := make(chan *structs.Chunk, 5)
38+
39+
input <- tt.args.input
40+
41+
conf := fecEncoderConfig{tt.args.required, tt.args.total, input, output}
42+
ctx, cancel := context.WithCancel(context.Background())
43+
go func() {
44+
time.Sleep(2 * time.Second)
45+
cancel()
46+
}()
47+
// ch <- tt.args.file
48+
worker(ctx, &conf)
49+
50+
if tt.wantErr {
51+
if !strings.Contains(memLog.String(), tt.expectedErr) {
52+
t.Fatalf("Expected not in log, '%v' not in '%v'", tt.expectedErr, memLog.String())
53+
}
54+
} else {
55+
for i := 0; i < conf.total; i++ {
56+
<-output
57+
}
58+
}
59+
})
60+
}
61+
}

‎pkg/filecloser/filecloser_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ func Test_worker(t *testing.T) {
122122
worker(ctx, &conf)
123123

124124
if !strings.Contains(memLog.String(), tt.expected) {
125-
t.Fatalf("Expected not in log, '%v' not in '%vs'", tt.expected, memLog.String())
125+
t.Fatalf("Expected not in log, '%v' not in '%v'", tt.expected, memLog.String())
126126
}
127127
})
128128
}

‎pkg/filereader/filereader_test.go‎

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -118,31 +118,8 @@ func Test_worker(t *testing.T) {
118118
worker(ctx, tt.args.conf)
119119

120120
if !strings.Contains(memLog.String(), tt.expected) {
121-
t.Fatalf("Expected not in log, '%v' not in '%vs'", tt.expected, memLog.String())
121+
t.Fatalf("Expected not in log, '%v' not in '%v'", tt.expected, memLog.String())
122122
}
123123
})
124124
}
125125
}
126-
127-
func TestCreateFileReader(t *testing.T) {
128-
type args struct {
129-
ctx context.Context
130-
db *gorm.DB
131-
chunksize int
132-
required int
133-
input chan database.File
134-
output chan *structs.Chunk
135-
workercount int
136-
}
137-
tests := []struct {
138-
name string
139-
args args
140-
}{
141-
// TODO: Add test cases.
142-
}
143-
for _, tt := range tests {
144-
t.Run(tt.name, func(t *testing.T) {
145-
CreateFileReader(tt.args.ctx, tt.args.db, tt.args.chunksize, tt.args.required, tt.args.input, tt.args.output, tt.args.workercount)
146-
})
147-
}
148-
}

‎pkg/sender/sender.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func Sender(ctx context.Context, db *gorm.DB, conf config.Config) {
2424

2525
queuereader.CreateQueueReader(ctx, db, queue_chan)
2626
filereader.CreateFileReader(ctx, db, conf.ChunkSize, conf.ChunkFecRequired, queue_chan, chunks_chan, maxprocs)
27-
fecencoder.CreateFecEncoder(ctx, conf.ChunkSize, conf.ChunkFecRequired, conf.ChunkFecTotal, chunks_chan, shares_chan, maxprocs)
27+
fecencoder.CreateFecEncoder(ctx, conf.ChunkFecRequired, conf.ChunkFecTotal, chunks_chan, shares_chan, maxprocs)
2828
bandwidthlimiter.CreateBandwidthLimiter(ctx, conf.BandwidthLimit, conf.ChunkSize, shares_chan, bw_limited_chunks, maxprocs)
2929
udpsender.CreateUdpSender(ctx, conf.ReceiverIP, conf.ReceiverPort, bw_limited_chunks, maxprocs)
3030
}

‎pkg/structs/structs.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"github.com/zhuangsirui/binpacker"
1313
)
1414

15-
const HASHSIZE = 32 // Using the sha256.Size as const directly causes linting issues
15+
const HASHSIZE = sha256.Size
1616

1717
func HashFile(f *os.File, encrypted bool) ([HASHSIZE]byte, error) {
1818
var ret [HASHSIZE]byte
@@ -35,7 +35,7 @@ func HashFile(f *os.File, encrypted bool) ([HASHSIZE]byte, error) {
3535

3636
type Chunk struct {
3737
Path string
38-
Hash [HASHSIZE]byte
38+
Hash [32]byte // Not using the HASHSIZE const as it causes linting issues
3939
Encrypted bool
4040
DataOffset int64
4141
DataPadding uint32

‎pkg/udpsender/udpsender_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package udpsender
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"crypto/rand"
7+
"math/big"
8+
"net"
9+
"oneway-filesync/pkg/structs"
10+
"strings"
11+
"testing"
12+
"time"
13+
14+
"github.com/sirupsen/logrus"
15+
)
16+
17+
func randint(max int64) int {
18+
nBig, err := rand.Int(rand.Reader, big.NewInt(max))
19+
if err != nil {
20+
panic(err)
21+
}
22+
return int(nBig.Int64())
23+
}
24+
25+
func Test_worker(t *testing.T) {
26+
ip := "127.0.0.1"
27+
port := randint(30000) + 30000
28+
addr := net.UDPAddr{
29+
IP: net.ParseIP(ip),
30+
Port: port,
31+
}
32+
33+
receiving_conn, err := net.ListenUDP("udp", &addr)
34+
if err != nil {
35+
t.Fatal(err)
36+
}
37+
defer receiving_conn.Close()
38+
39+
type args struct {
40+
ip string
41+
port int
42+
chunk structs.Chunk
43+
}
44+
tests := []struct {
45+
name string
46+
args args
47+
wantErr bool
48+
expectedErr string
49+
}{
50+
{"test-works", args{ip, port, structs.Chunk{}}, false, ""},
51+
{"test-socket-err", args{ip, 88888, structs.Chunk{}}, true, "Error creating udp socket"},
52+
{"test-message-too-long", args{ip, port, structs.Chunk{Data: make([]byte, 100*1024)}}, true, "Error sending share: write udp"},
53+
}
54+
for _, tt := range tests {
55+
t.Run(tt.name, func(t *testing.T) {
56+
var memLog bytes.Buffer
57+
logrus.SetOutput(&memLog)
58+
59+
input := make(chan *structs.Chunk, 5)
60+
input <- &tt.args.chunk
61+
conf := udpSenderConfig{tt.args.ip, tt.args.port, input}
62+
ctx, cancel := context.WithCancel(context.Background())
63+
go func() {
64+
time.Sleep(2 * time.Second)
65+
cancel()
66+
}()
67+
worker(ctx, &conf)
68+
69+
if tt.wantErr {
70+
if !strings.Contains(memLog.String(), tt.expectedErr) {
71+
t.Fatalf("Expected not in log, '%v' not in '%v'", tt.expectedErr, memLog.String())
72+
}
73+
} else {
74+
_, _ = receiving_conn.Read(make([]byte, 8192))
75+
}
76+
})
77+
}
78+
}

‎pkg/watcher/watcher.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func worker(ctx context.Context, conf *watcherConfig) {
5050
delete(conf.cache, path)
5151
err := database.QueueFileForSending(conf.db, path, conf.encrypted)
5252
if err != nil {
53-
logrus.Errorf("%v", err)
53+
logrus.Errorf("Failed to queue file for sending: %v", err)
5454
} else {
5555
logrus.Infof("File '%s' queued for sending", path)
5656
}

0 commit comments

Comments
 (0)