Merge branch 'develop' into backend/client-broker

Author: jmaralo

.github/workflows/build-backend.yaml

@@ -6,6 +6,9 @@ on:
       SECRET:
         required: true
   workflow_dispatch:
+  pull_request:
+    paths:
+      - backend/**
 
 env:
   SECRET: ${{ secrets.SECRET }}
@@ -26,7 +29,9 @@ jobs:
     - name: "Install packages"
       run: apk update && apk add --no-cache libpcap-dev musl-dev gcc go
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+      with:
+        sparse-checkout: backend
 
     - name: "Create output path"
       working-directory: "${{env.BACKEND_DIR}}"
@@ -47,10 +52,12 @@ jobs:
         go build -ldflags '-linkmode external -extldflags "-static"' -o ../output/backend-linux-64
 
     - name: "Upload build"
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: backend-linux
         path: "${{env.BACKEND_DIR}}/output/*"
+        retention-days: 3
+        compression-level: 9
 
   build-backend-windows:
     name: "Build backend for windows"
@@ -90,10 +97,12 @@ jobs:
         go build -o ..\output\backend-windows-64.exe
 
     - name: "Upload build"
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: backend-windows
         path: "${{env.BACKEND_DIR}}\\output\\*"
+        retention-days: 3
+        compression-level: 9
 
   build-backend-mac:
     name: "Build backend for macOS"
@@ -142,7 +151,9 @@ jobs:
         go build -o ../output/backend-macos-m1-64
       
     - name: "Upload build"
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: backend-macos
         path: "${{env.BACKEND_DIR}}/output/*"
+        retention-days: 3
+        compression-level: 9

.github/workflows/build-ethernet-view.yaml

@@ -3,6 +3,10 @@ name: Build ethernet view
 on:
   workflow_call:
   workflow_dispatch:
+  pull_request:
+    paths:
+      - ethernet-view/**
+      - common-front/**
 
 jobs:
   build-ethernet-view:
@@ -14,7 +18,11 @@ jobs:
       COMMON_DIR: ./common-front
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+      with:
+        sparse-checkout: |
+          ethernet-view
+          common-front
 
     - name: "Install common front dependencies"
       working-directory: "${{env.COMMON_DIR}}"
@@ -36,7 +44,9 @@ jobs:
 
 
     - name: "Upload build"
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ethernet-view
         path: "${{env.FRONTEND_DIR}}/static/*"
+        retention-days: 3
+        compression-level: 9

.github/workflows/test-backend.yaml

@@ -0,0 +1,45 @@
+name: Test backend
+
+on:
+    push:
+      paths:
+        - backend/**
+    pull_request:
+      paths:
+        - backend/**
+    workflow_dispatch:
+
+env:
+  SECRET: ${{ secrets.SECRET }}
+
+jobs:
+  test-backend:
+    name: "Test backend"
+    runs-on: ubuntu-latest
+
+    env:
+      BACKEND_DIR: ./backend
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21
+          cache: false
+          
+
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y libpcap-dev
+
+      - name: "Load secret"
+        working-directory: "${{env.BACKEND_DIR}}"
+        run: |
+          ./load-secret.sh "$SECRET"
+
+      - name: Test with Go
+        working-directory: "${{env.BACKEND_DIR}}"
+        run: go test -v -timeout 30s ./...

.gitignore

@@ -1,4 +1,5 @@
 build
+.vscode
 
 # GOOGLE API KEY
 secret.json

backend/cmd/config.toml

@@ -21,8 +21,6 @@ udp_tag = "UDP"
 mtu = 1500
 interface = "lo"
 # blcu_ack_id = "blcu_ack"
-keep_alive_interval = "1s"
-keep_alive_probes = 3
 timeout = "1s"
 
 [vehicle.messages]

backend/cmd/main.go

@@ -22,23 +22,18 @@ import (
 	"github.com/HyperloopUPV-H8/h9-backend/internal/excel"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/excel/ade"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/info"
-	"github.com/HyperloopUPV-H8/h9-backend/internal/logger_handler"
-	protection_logger "github.com/HyperloopUPV-H8/h9-backend/internal/message_logger"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/message_transfer"
-	"github.com/HyperloopUPV-H8/h9-backend/internal/order_logger"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/order_transfer"
-	"github.com/HyperloopUPV-H8/h9-backend/internal/packet_logger"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/pod_data"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/server"
-	"github.com/HyperloopUPV-H8/h9-backend/internal/state_space_logger"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/update_factory"
-	"github.com/HyperloopUPV-H8/h9-backend/internal/value_logger"
 	vehicle_models "github.com/HyperloopUPV-H8/h9-backend/internal/vehicle/models"
 	"github.com/HyperloopUPV-H8/h9-backend/internal/ws_handle"
 	"github.com/HyperloopUPV-H8/h9-backend/pkg/abstraction"
 	"github.com/HyperloopUPV-H8/h9-backend/pkg/broker"
 	connection_topic "github.com/HyperloopUPV-H8/h9-backend/pkg/broker/topics/connection"
 	data_topic "github.com/HyperloopUPV-H8/h9-backend/pkg/broker/topics/data"
+	"github.com/HyperloopUPV-H8/h9-backend/pkg/excel/utils"
 	"github.com/HyperloopUPV-H8/h9-backend/pkg/transport"
 	"github.com/HyperloopUPV-H8/h9-backend/pkg/transport/network/sniffer"
 	"github.com/HyperloopUPV-H8/h9-backend/pkg/transport/network/tcp"
@@ -55,6 +50,12 @@ import (
 	"github.com/google/gopacket/pcap"
 	"github.com/pelletier/go-toml/v2"
 	trace "github.com/rs/zerolog/log"
+
+	"github.com/HyperloopUPV-H8/h9-backend/pkg/logger"
+	data_logger "github.com/HyperloopUPV-H8/h9-backend/pkg/logger/data"
+	messages_logger "github.com/HyperloopUPV-H8/h9-backend/pkg/logger/messages"
+	order_logger "github.com/HyperloopUPV-H8/h9-backend/pkg/logger/order"
+	state_logger "github.com/HyperloopUPV-H8/h9-backend/pkg/logger/state"
 )
 
 var traceLevel = flag.String("trace", "info", "set the trace level (\"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\")")
@@ -128,20 +129,15 @@ func main() {
 	updateFactory := update_factory.NewFactory()
 
 	// <--- logger --->
-	packetLogger := packet_logger.NewPacketLogger(podData.Boards, config.PacketLogger)
-	valueLogger := value_logger.NewValueLogger(podData.Boards, config.ValueLogger)
-	orderLogger := order_logger.NewOrderLogger(podData.Boards, config.OrderLogger)
-	protectionLogger := protection_logger.NewMessageLogger(config.Vehicle.Messages.InfoIdKey, config.Vehicle.Messages.FaultIdKey, config.Vehicle.Messages.WarningIdKey, config.ProtectionLogger)
-	stateSpaceLogger := state_space_logger.NewStateSpaceLogger(info.MessageIds.StateSpace)
-
-	loggers := map[string]logger_handler.Logger{
-		"packets":     &packetLogger,
-		"values":      &valueLogger,
-		"orders":      &orderLogger,
-		"protections": &protectionLogger,
-		"stateSpace":  &stateSpaceLogger,
+	var boardMap map[abstraction.BoardId]string
+	var subloggers = map[abstraction.LoggerName]abstraction.Logger{
+		data_logger.Name:     data_logger.NewLogger(),
+		messages_logger.Name: messages_logger.NewLogger(boardMap),
+		order_logger.Name:    order_logger.NewLogger(),
+		state_logger.Name:    state_logger.NewLogger(),
 	}
-	loggerHandler := logger_handler.NewLoggerHandler(loggers, config.LoggerHandler)
+
+	loggerHandler := logger.NewLogger(subloggers)
 
 	// <--- order transfer --->
 	idToBoard := make(map[uint16]string)
@@ -189,40 +185,67 @@ func main() {
 	transp.SetAPI(&transportAPI{
 		OnNotification: func(notification abstraction.TransportNotification) {
 			packet := notification.(transport.PacketNotification)
+
 			switch p := packet.Packet.(type) {
 			case *data.Packet:
-				if _, ok := orders[p.Id()]; ok {
-					loggerHandler.Log(order_logger.LoggableOrder(*p))
-					return
-				}
-
 				update := updateFactory.NewUpdate(p)
 				err := broker.Push(data_topic.NewPush(&update))
 				if err != nil {
 					fmt.Println(err)
 				}
 
-				loggerHandler.Log(packet_logger.ToLoggablePacket(p))
+				err = loggerHandler.PushRecord(&data_logger.Record{
+					Packet: p,
+				})
 
-				for id, value := range p.GetValues() {
-					loggerHandler.Log(value_logger.ToLoggableValue(string(id), value, p.Timestamp()))
+				if err != nil {
+					fmt.Println("Error pushing record to logger: ", err)
 				}
+
 			case *info_packet.Packet:
 				messageTransfer.SendMessage(p)
-				loggerHandler.Log(protection_logger.LoggableInfo(*p))
+
+				err = loggerHandler.PushRecord(&messages_logger.Record{
+					Packet: p,
+				})
+
+				if err != nil {
+					fmt.Println("Error pushing record to logger: ", err)
+				}
+
 			case *protection.Packet:
 				messageTransfer.SendMessage(p)
-				loggerHandler.Log(protection_logger.LoggableProtection(*p))
+
+				packet := info_packet.NewPacket(p.Id())
+				packet.BoardId = p.BoardId
+				packet.Timestamp = p.Timestamp
+				packet.Msg = info_packet.InfoData(fmt.Sprint(p))
+
+				err = loggerHandler.PushRecord(&messages_logger.Record{
+					Packet: packet,
+				})
+
+				if err != nil {
+					fmt.Println("Error pushing record to logger: ", err)
+				}
+
 			case *blcu_packet.Ack:
 				if useBlcu {
 					blcu.NotifyAck()
 				}
+
 			case *state.Space:
-				for _, row := range p.State() {
-					loggerHandler.Log(state_space_logger.LoggableStateSpaceRow(row))
+				err = loggerHandler.PushRecord(&state_logger.Record{
+					Packet: p,
+				})
+
+				if err != nil {
+					fmt.Println("Error pushing record to logger: ", err)
 				}
+
 			case *order.Add:
 				orderTransfer.AddStateOrders(*p)
+
 			case *order.Remove:
 				orderTransfer.RemoveStateOrders(*p)
 			}
@@ -284,7 +307,13 @@ func main() {
 				trace.Error().Any("order", order).Err(err).Msg("error sending order")
 			}
 
-			loggerHandler.Log(order_logger.LoggableOrder(order))
+			err = loggerHandler.PushRecord(&order_logger.Record{
+				Packet: &order,
+			})
+
+			if err != nil {
+				fmt.Println("Error pushing record to logger: ", err)
+			}
 		}
 	}()
 
@@ -310,7 +339,7 @@ func main() {
 
 	websocketBroker.RegisterHandle(&connectionTransfer, config.Connections.UpdateTopic, "connection/update")
 	websocketBroker.RegisterHandle(&dataTransfer, "podData/update")
-	websocketBroker.RegisterHandle(&loggerHandler, config.LoggerHandler.Topics.Enable)
+	websocketBroker.RegisterHandle(loggerHandler, config.LoggerHandler.Topics.Enable)
 	websocketBroker.RegisterHandle(&messageTransfer, "message/update")
 	websocketBroker.RegisterHandle(&orderTransfer, config.Orders.SendTopic, "order/stateOrders")
 
@@ -462,27 +491,29 @@ func getTransportDecEnc(info info.Info, podData pod_data.PodData) (*presentation
 			for i, measurement := range packet.Measurements {
 				switch meas := measurement.(type) {
 				case pod_data.NumericMeasurement:
+					podOps := getOps(meas.PodUnits)
+					displayOps := getOps(meas.DisplayUnits)
 					switch meas.Type {
 					case "uint8":
-						descriptor[i] = data.NewNumericDescriptor[uint8](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[uint8](data.ValueName(meas.Id), podOps, displayOps)
 					case "uint16":
-						descriptor[i] = data.NewNumericDescriptor[uint16](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[uint16](data.ValueName(meas.Id), podOps, displayOps)
 					case "uint32":
-						descriptor[i] = data.NewNumericDescriptor[uint32](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[uint32](data.ValueName(meas.Id), podOps, displayOps)
 					case "uint64":
-						descriptor[i] = data.NewNumericDescriptor[uint64](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[uint64](data.ValueName(meas.Id), podOps, displayOps)
 					case "int8":
-						descriptor[i] = data.NewNumericDescriptor[int8](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[int8](data.ValueName(meas.Id), podOps, displayOps)
 					case "int16":
-						descriptor[i] = data.NewNumericDescriptor[int16](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[int16](data.ValueName(meas.Id), podOps, displayOps)
 					case "int32":
-						descriptor[i] = data.NewNumericDescriptor[int32](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[int32](data.ValueName(meas.Id), podOps, displayOps)
 					case "int64":
-						descriptor[i] = data.NewNumericDescriptor[int64](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[int64](data.ValueName(meas.Id), podOps, displayOps)
 					case "float32":
-						descriptor[i] = data.NewNumericDescriptor[float32](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[float32](data.ValueName(meas.Id), podOps, displayOps)
 					case "float64":
-						descriptor[i] = data.NewNumericDescriptor[float64](data.ValueName(meas.Id))
+						descriptor[i] = data.NewNumericDescriptor[float64](data.ValueName(meas.Id), podOps, displayOps)
 					default:
 						panic(fmt.Sprintf("unexpected numeric type for %s: %s", meas.Id, meas.Type))
 					}
@@ -526,6 +557,17 @@ func getTransportDecEnc(info info.Info, podData pod_data.PodData) (*presentation
 	return decoder, encoder
 }
 
+func getOps(units utils.Units) data.ConversionDescriptor {
+	output := make(data.ConversionDescriptor, len(units.Operations))
+	for i, operation := range units.Operations {
+		output[i] = data.Operation{
+			Operator: operation.Operator,
+			Operand:  operation.Operand,
+		}
+	}
+	return output
+}
+
 type transportAPI struct {
 	OnNotification     func(abstraction.TransportNotification)
 	OnConnectionUpdate func(abstraction.TransportTarget, bool)

backend/pkg/abstraction/logger.go

@@ -16,8 +16,8 @@ type LoggerRequest interface {
 // Logger is the module in charge of storing and retrieving information from the
 // hard drive, making it persistent.
 type Logger interface {
-	Start(startKeys []LoggerName) error
-	Stop(stopKeys []LoggerName) error
+	Start() error
+	Stop() error
 	// PushRecord will store a record to disk
 	PushRecord(LoggerRecord) error
 	// PullRecord will retrieve a record from disk