Skip to content

Commit e745867

Browse files
authored
Merge pull request #150 from HyperloopUPV-H8/develop
Merge optimizations
2 parents cb3818c + b79759f commit e745867

145 files changed

Lines changed: 3293 additions & 2651 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
build
2+
profiles
23

34
# GOOGLE API KEY
45
secret.json

backend/cmd/main.go

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"fmt"
99
"log"
1010
"net"
11+
"net/http"
12+
_ "net/http/pprof"
1113
"os"
1214
"os/signal"
1315
"path"
@@ -59,6 +61,8 @@ var traceLevel = flag.String("trace", "info", "set the trace level (\"fatal\", \
5961
var traceFile = flag.String("log", "trace.json", "set the trace log file")
6062
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
6163
var enableSNTP = flag.Bool("sntp", false, "enables a simple SNTP server on port 123")
64+
var networkDevice = flag.Int("dev", -1, "index of the network device to use, overrides device prompt")
65+
var blockprofile = flag.Int("blockprofile", 0, "number of block profiles to include")
6266

6367
func main() {
6468
flag.Parse()
@@ -79,6 +83,7 @@ func main() {
7983
pprof.StartCPUProfile(f)
8084
defer pprof.StopCPUProfile()
8185
}
86+
runtime.SetBlockProfileRate(*blockprofile)
8287
config := getConfig("./config.toml")
8388

8489
file, err := excel.Download(excel.DownloadConfig(config.Excel.Download))
@@ -102,10 +107,19 @@ func main() {
102107
trace.Fatal().Err(err).Msg("creating podData")
103108
}
104109

105-
dev, err := selectDev()
106-
if err != nil {
107-
trace.Fatal().Err(err).Msg("Error selecting device")
108-
panic(err)
110+
var dev pcap.Interface
111+
if *networkDevice != -1 {
112+
devs, err := pcap.FindAllDevs()
113+
if err != nil {
114+
trace.Fatal().Err(err).Msg("Getting devices")
115+
}
116+
117+
dev = devs[*networkDevice]
118+
} else {
119+
dev, err = selectDev()
120+
if err != nil {
121+
trace.Fatal().Err(err).Msg("Error selecting device")
122+
}
109123
}
110124

111125
vehicleOrders, err := vehicle_models.NewVehicleOrders(podData.Boards, config.Excel.Parse.Global.BLCUAddressKey)
@@ -276,6 +290,8 @@ func main() {
276290
go httpServer.ListenAndServe()
277291
}
278292

293+
go http.ListenAndServe("127.0.0.1:4040", nil)
294+
279295
// <--- SNTP --->
280296
if *enableSNTP {
281297
sntpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", info.Addresses.Backend, info.Ports.SNTP))
@@ -480,13 +496,13 @@ func getTransportDecEnc(info info.Info, podData pod_data.PodData) (*presentation
480496
decoder.SetPacketDecoder(abstraction.PacketId(info.MessageIds.RemoveStateOrder), stateOrdersDecoder)
481497

482498
protectionDecoder := protection.NewDecoder(binary.LittleEndian)
483-
protectionDecoder.SetSeverity(1000, protection.Fault).SetSeverity(2000, protection.Warning).SetSeverity(3000, protection.Ok)
484-
protectionDecoder.SetSeverity(1111, protection.Fault).SetSeverity(2111, protection.Warning).SetSeverity(3111, protection.Ok)
485-
protectionDecoder.SetSeverity(1222, protection.Fault).SetSeverity(2222, protection.Warning).SetSeverity(3222, protection.Ok)
486-
protectionDecoder.SetSeverity(1333, protection.Fault)
487-
protectionDecoder.SetSeverity(1444, protection.Fault)
488-
protectionDecoder.SetSeverity(1555, protection.Fault)
489-
protectionDecoder.SetSeverity(1666, protection.Fault).SetSeverity(2666, protection.Warning).SetSeverity(3666, protection.Ok)
499+
protectionDecoder.SetSeverity(1000, protection.FaultSeverity).SetSeverity(2000, protection.WarningSeverity).SetSeverity(3000, protection.OkSeverity)
500+
protectionDecoder.SetSeverity(1111, protection.FaultSeverity).SetSeverity(2111, protection.WarningSeverity).SetSeverity(3111, protection.OkSeverity)
501+
protectionDecoder.SetSeverity(1222, protection.FaultSeverity).SetSeverity(2222, protection.WarningSeverity).SetSeverity(3222, protection.OkSeverity)
502+
protectionDecoder.SetSeverity(1333, protection.FaultSeverity)
503+
protectionDecoder.SetSeverity(1444, protection.FaultSeverity)
504+
protectionDecoder.SetSeverity(1555, protection.FaultSeverity).SetSeverity(2555, protection.WarningSeverity)
505+
protectionDecoder.SetSeverity(1666, protection.FaultSeverity).SetSeverity(2666, protection.WarningSeverity).SetSeverity(3666, protection.OkSeverity)
490506
decoder.SetPacketDecoder(1000, protectionDecoder)
491507
decoder.SetPacketDecoder(1111, protectionDecoder)
492508
decoder.SetPacketDecoder(1222, protectionDecoder)
@@ -497,6 +513,7 @@ func getTransportDecEnc(info info.Info, podData pod_data.PodData) (*presentation
497513
decoder.SetPacketDecoder(2000, protectionDecoder)
498514
decoder.SetPacketDecoder(2111, protectionDecoder)
499515
decoder.SetPacketDecoder(2222, protectionDecoder)
516+
decoder.SetPacketDecoder(2555, protectionDecoder)
500517
decoder.SetPacketDecoder(2666, protectionDecoder)
501518
decoder.SetPacketDecoder(3000, protectionDecoder)
502519
decoder.SetPacketDecoder(3111, protectionDecoder)

backend/pkg/broker/topics/message/message_test.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ package message_test
22

33
import (
44
"encoding/json"
5+
"os"
6+
"testing"
7+
"time"
8+
59
"github.com/HyperloopUPV-H8/h9-backend/pkg/abstraction"
610
"github.com/HyperloopUPV-H8/h9-backend/pkg/broker"
711
data "github.com/HyperloopUPV-H8/h9-backend/pkg/broker/topics/message"
@@ -10,9 +14,6 @@ import (
1014
"github.com/HyperloopUPV-H8/h9-backend/pkg/websocket"
1115
ws "github.com/gorilla/websocket"
1216
"github.com/rs/zerolog"
13-
"os"
14-
"testing"
15-
"time"
1617
)
1718

1819
func TestMessageTopic_Push(t *testing.T) {
@@ -82,7 +83,7 @@ func TestMessageTopic_ClientMessage(t *testing.T) {
8283
messageTopic := data.NewUpdateTopic(map[abstraction.BoardId]string{})
8384
messageTopic.SetAPI(api)
8485

85-
packet := protection.NewPacket(0, protection.Ok)
86+
packet := protection.NewPacket(0, protection.OkSeverity)
8687
payload := data.Push(packet, 0)
8788
payloadBytes, _ := json.Marshal(payload)
8889
messageTopic.ClientMessage(websocket.ClientId{0}, &websocket.Message{

backend/pkg/logger/data/logger.go

Lines changed: 61 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package data
22

33
import (
4-
"encoding/csv"
54
"fmt"
6-
"io"
75
"os"
86
"path"
97
"strconv"
@@ -13,6 +11,7 @@ import (
1311

1412
"github.com/HyperloopUPV-H8/h9-backend/pkg/abstraction"
1513
loggerHandler "github.com/HyperloopUPV-H8/h9-backend/pkg/logger"
14+
"github.com/HyperloopUPV-H8/h9-backend/pkg/logger/file"
1615
"github.com/HyperloopUPV-H8/h9-backend/pkg/transport/packet/data"
1716
)
1817

@@ -25,8 +24,8 @@ type Logger struct {
2524
// An atomic boolean is used in order to use CompareAndSwap in the Start and Stop methods
2625
running *atomic.Bool
2726
fileLock *sync.RWMutex
28-
// valueFileSlice is a map that contains the file of each value
29-
valueFileSlice map[data.ValueName]io.WriteCloser
27+
// saveFiles is a map that contains the file of each value
28+
saveFiles map[data.ValueName]*file.CSV
3029
}
3130

3231
// Record is a struct that implements the abstraction.LoggerRecord interface
@@ -41,9 +40,9 @@ func (*Record) Name() abstraction.LoggerName { return Name }
4140

4241
func NewLogger() *Logger {
4342
logger := &Logger{
44-
valueFileSlice: make(map[data.ValueName]io.WriteCloser),
45-
running: &atomic.Bool{},
46-
fileLock: &sync.RWMutex{},
43+
saveFiles: make(map[data.ValueName]*file.CSV),
44+
running: &atomic.Bool{},
45+
fileLock: &sync.RWMutex{},
4746
}
4847

4948
logger.running.Store(false)
@@ -83,75 +82,76 @@ func (sublogger *Logger) PushRecord(record abstraction.LoggerRecord) error {
8382
}
8483
}
8584

86-
valueMap := dataRecord.Packet.GetValues()
85+
writeErr := error(nil)
86+
for valueName, value := range dataRecord.Packet.GetValues() {
8787

88-
sublogger.fileLock.Lock()
89-
defer sublogger.fileLock.Unlock()
90-
91-
writerErr := error(nil)
92-
for valueName, value := range valueMap {
93-
94-
timestamp := dataRecord.Packet.Timestamp()
95-
96-
var val string
97-
98-
switch v := value.(type) {
88+
var valueRepresentation string
89+
switch value := value.(type) {
9990
case numeric:
100-
val = strconv.FormatFloat(v.Value(), 'f', -1, 64)
101-
91+
valueRepresentation = strconv.FormatFloat(value.Value(), 'f', -1, 64)
10292
case data.BooleanValue:
103-
val = strconv.FormatBool(v.Value())
104-
93+
valueRepresentation = strconv.FormatBool(value.Value())
10594
case data.EnumValue:
106-
val = string(v.Variant())
95+
valueRepresentation = string(value.Variant())
10796
}
10897

109-
file, ok := sublogger.valueFileSlice[valueName]
110-
if !ok {
111-
filename := path.Join(
112-
"logger", "data",
113-
loggerHandler.Timestamp.Format(loggerHandler.TimestampFormat),
114-
fmt.Sprintf("%s.csv", valueName),
115-
)
116-
err := os.MkdirAll(path.Dir(filename), os.ModePerm)
117-
if err != nil {
118-
return loggerHandler.ErrCreatingAllDir{
119-
Name: Name,
120-
Timestamp: time.Now(),
121-
Path: filename,
122-
}
123-
}
124-
125-
f, err := os.Create(path.Join(filename))
126-
if err != nil {
127-
return loggerHandler.ErrCreatingFile{
128-
Name: Name,
129-
Timestamp: time.Now(),
130-
Inner: err,
131-
}
132-
}
133-
sublogger.valueFileSlice[valueName] = f
134-
file = f
98+
saveFile, err := sublogger.getFile(valueName)
99+
if err != nil {
100+
return err
135101
}
136-
writer := csv.NewWriter(file) // TODO! use map/slice of writer
137102

138-
err := writer.Write([]string{
139-
fmt.Sprint(timestamp.UnixMilli()),
103+
err = saveFile.Write([]string{
104+
fmt.Sprint(dataRecord.Packet.Timestamp().UnixMilli()),
140105
dataRecord.From,
141106
dataRecord.To,
142-
val,
107+
valueRepresentation,
143108
})
109+
saveFile.Flush()
110+
144111
if err != nil {
145-
writerErr = loggerHandler.ErrWritingFile{
112+
writeErr = loggerHandler.ErrWritingFile{
146113
Name: Name,
147114
Timestamp: time.Now(),
148115
Inner: err,
149116
}
150-
fmt.Println(writerErr)
117+
fmt.Println(writeErr)
151118
}
152-
writer.Flush()
153119
}
154-
return writerErr
120+
return writeErr
121+
}
122+
123+
func (sublogger *Logger) getFile(valueName data.ValueName) (*file.CSV, error) {
124+
sublogger.fileLock.Lock()
125+
defer sublogger.fileLock.Unlock()
126+
127+
valueFile, ok := sublogger.saveFiles[valueName]
128+
if ok {
129+
return valueFile, nil
130+
}
131+
132+
valueFileRaw, err := sublogger.createFile(valueName)
133+
sublogger.saveFiles[valueName] = file.NewCSV(valueFileRaw)
134+
135+
return sublogger.saveFiles[valueName], err
136+
}
137+
138+
func (sublogger *Logger) createFile(valueName data.ValueName) (*os.File, error) {
139+
filename := path.Join(
140+
"logger", "data",
141+
loggerHandler.Timestamp.Format(loggerHandler.TimestampFormat),
142+
fmt.Sprintf("%s.csv", valueName),
143+
)
144+
145+
err := os.MkdirAll(path.Dir(filename), os.ModePerm)
146+
if err != nil {
147+
return nil, loggerHandler.ErrCreatingAllDir{
148+
Name: Name,
149+
Timestamp: time.Now(),
150+
Path: filename,
151+
}
152+
}
153+
154+
return os.Create(path.Join(filename))
155155
}
156156

157157
func (sublogger *Logger) PullRecord(abstraction.LoggerRequest) (abstraction.LoggerRecord, error) {
@@ -165,19 +165,19 @@ func (sublogger *Logger) Stop() error {
165165
}
166166

167167
closeErr := error(nil)
168-
for value, file := range sublogger.valueFileSlice {
168+
for valueName, file := range sublogger.saveFiles {
169169
err := file.Close()
170170
if err != nil {
171171
closeErr = loggerHandler.ErrClosingFile{
172172
Name: Name,
173173
Timestamp: time.Now(),
174174
}
175175

176-
fmt.Println(value, ": ", closeErr)
176+
fmt.Println(valueName, ": ", closeErr)
177177
}
178178
}
179179

180-
sublogger.valueFileSlice = make(map[data.ValueName]io.WriteCloser, len(sublogger.valueFileSlice))
180+
sublogger.saveFiles = make(map[data.ValueName]*file.CSV, len(sublogger.saveFiles))
181181

182182
fmt.Println("Logger stopped")
183183
return closeErr

backend/pkg/logger/file/csv.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package file
2+
3+
import (
4+
"encoding/csv"
5+
"os"
6+
"sync"
7+
)
8+
9+
// CSV is a CSV file representation that is safe to read and write concurrently.
10+
type CSV struct {
11+
lock *sync.Mutex
12+
file *os.File
13+
writer *csv.Writer
14+
reader *csv.Reader
15+
}
16+
17+
func NewCSV(file *os.File) *CSV {
18+
return &CSV{
19+
lock: new(sync.Mutex),
20+
file: file,
21+
writer: csv.NewWriter(file),
22+
reader: csv.NewReader(file),
23+
}
24+
}
25+
26+
func (file *CSV) Write(record []string) error {
27+
file.lock.Lock()
28+
defer file.lock.Unlock()
29+
return file.writer.Write(record)
30+
}
31+
32+
func (file *CSV) Read() (record []string, err error) {
33+
file.lock.Lock()
34+
defer file.lock.Unlock()
35+
return file.reader.Read()
36+
}
37+
38+
func (file *CSV) Flush() {
39+
file.lock.Lock()
40+
defer file.lock.Unlock()
41+
file.writer.Flush()
42+
}
43+
44+
func (file *CSV) Close() error {
45+
file.Flush()
46+
return file.file.Close()
47+
}

backend/pkg/logger/logger_test.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ package logger_test
33
import (
44
"encoding/csv"
55
"fmt"
6+
"os"
7+
"path"
8+
"testing"
9+
"time"
10+
611
"github.com/HyperloopUPV-H8/h9-backend/pkg/abstraction"
712
"github.com/HyperloopUPV-H8/h9-backend/pkg/logger"
813
"github.com/HyperloopUPV-H8/h9-backend/pkg/logger/data"
@@ -13,10 +18,6 @@ import (
1318
protectionPacketer "github.com/HyperloopUPV-H8/h9-backend/pkg/transport/packet/protection"
1419
statePacketer "github.com/HyperloopUPV-H8/h9-backend/pkg/transport/packet/state"
1520
"github.com/rs/zerolog"
16-
"os"
17-
"path"
18-
"testing"
19-
"time"
2021
)
2122

2223
func TestLogger(t *testing.T) {
@@ -115,7 +116,7 @@ func TestLogger(t *testing.T) {
115116
}
116117

117118
// Protection
118-
protectionPacket := protectionPacketer.NewPacket(0, protectionPacketer.Ok)
119+
protectionPacket := protectionPacketer.NewPacket(0, protectionPacketer.OkSeverity)
119120
protectionPacket.Timestamp = &protectionPacketer.Timestamp{
120121
Counter: 0,
121122
Second: 0,

0 commit comments

Comments
 (0)