Telemetry via serial port (#97)

* add frontend visual serial port selector and remove unnecessary prints

* Make time out work

* optimize parse data for lossy telemetry data

---------

Co-authored-by: sbasu107 <40325803+sbasu107@users.noreply.github.com>

Author: MCLiii

Backend/.gitignore

@@ -9,3 +9,6 @@ recordedData/processedData/*.csv
 
 # Python
 *.egg-info
+
+# File sync
+Downloaded*/

Backend/core/comms.py

@@ -5,6 +5,7 @@
 import aiohttp
 import asyncio
 import config
+import serial
 
 import signal
 import sys
@@ -13,15 +14,17 @@
 from multiprocessing.managers import BaseManager
 from . import db
 from file_sync.file_sync_down.main import *
+import re
 
 format_string = '<' # little-endian
 byte_length = 0
 properties = []
 frontend_data = {}
-solar_car_connection = {'lte': False, 'udp': False}
+solar_car_connection = {'lte': False, 'udp': False, 'serial': False}
 # Convert dataformat to format string for struct conversion
 # Docs: https://docs.python.org/3/library/struct.html
 types = {'bool': '?', 'float': 'f', 'char': 'c', 'uint8': 'B', 'uint16': 'H', 'uint64': 'Q'}
+serial_port = {"device": "", 'baud': 115200}    # shared object with core_api for setting serial device from frontend
 
 def set_format(file_path: str):
     global format_string, byte_length, properties
@@ -46,7 +49,7 @@ def unpack_data(data):
 
 
 class Telemetry:
-    __tmp_data = {'tcp': b'', 'lte': b'', 'udp': b'', 'file_sync': b''}
+    __tmp_data = {'tcp': b'', 'lte': b'', 'udp': b'', 'file_sync': b'', 'serial': b''}
     latest_tstamp = 0
 
     def listen_udp(self, port: int):
@@ -141,6 +144,51 @@ def listen_tcp(self, server_addr: str, port: int):
                     solar_car_connection['tcp'] = False
                     break
 
+    def serial_read(self):
+        global frontend_data, serial_port
+        latest_tstamp = 0
+        while True:
+            curr_device = serial_port['device']
+            curr_baud = serial_port['baud']
+            if(curr_device):
+                # Establish a serial connection)
+                ser = serial.Serial(curr_device, curr_baud)
+                # if device has been updated then exit loop and connect to new device
+                while curr_device == serial_port['device'] and curr_baud == serial_port['baud']:
+                    if time.time() - latest_tstamp > 5:
+                        solar_car_connection['serial'] = False
+                    # Read data from serial port
+                    try:
+                        data = b''
+                        if(ser.in_waiting > 0):
+                            data = ser.read(ser.in_waiting)
+                        else:
+                            time.sleep(0.1)
+                        if not data:
+                            # No data received, continue listening
+                            continue
+                        packets = self.parse_packets(data, 'serial')
+                        for packet in packets:
+                            if len(packet) == byte_length:
+                                d = unpack_data(packet)
+                                latest_tstamp = time.time()
+                                try:
+                                    frontend_data = d.copy()
+                                    db.insert_data(d)
+                                except Exception as e:
+                                    print(traceback.format_exc())
+                                    continue
+                                solar_car_connection['serial'] = True
+                    except Exception:
+                        print(traceback.format_exc())
+                        solar_car_connection['serial'] = False
+                        serial_port['device'] = ""
+                        break
+            else:
+                solar_car_connection['serial'] = False
+                # wait before retry
+                time.sleep(1)
+
     async def fetch(self, session, url):
         try:
             async with session.get(url, timeout=2) as response:
@@ -195,35 +243,40 @@ async def remote_db_fetch(self, server_url: str):
 
     def parse_packets(self, new_data: bytes, tmp_source: str):
         """
-        Parse and check the length of each packet
-        :param new_data: Newly received bytes from the comm channel
-        :param tmp_source: Name of tmp data source, put comm channel name here e.g. tcp, lte
+        Parse and check the length of each packet.
+        
+        :param new_data: Newly received bytes from the comm channel.
+        :param tmp_source: Name of tmp data source, put comm channel name here e.g. tcp, lte.
         """
-        header = b'<bsr>'
-        footer = b'</bsr>'
+        header = b"<bsr>"
+        footer = b"<bsr"
+        if tmp_source not in self.__tmp_data:
+            self.__tmp_data[tmp_source] = b''
+
+        # Append new data to the temporary buffer
         self.__tmp_data[tmp_source] += new_data
+
+        # Regex pattern to match packets with <bsr> and </bsr> tags
+        pattern = re.compile(b'<bsr>(.*?)</bsr>', re.DOTALL)
+
         packets = []
         while True:
-            # Search for the next complete data packet
-            try:
-                start_index = self.__tmp_data[tmp_source].index(header)
-                end_index = self.__tmp_data[tmp_source].index(footer)
-            except ValueError:
+            match = pattern.search(self.__tmp_data[tmp_source])
+            if not match:
                 break
+            # Extract the packet data
+            packet = match.group(1)
+            #remove headers and footers
+            packets.append(packet)
 
-            # Extract a complete data packet
-            packets.append(self.__tmp_data[tmp_source][start_index + len(header):end_index])
-            # Update the remaining data to exclude the processed packet
-            self.__tmp_data[tmp_source] = self.__tmp_data[tmp_source][end_index + len(footer):]
-
-        # If the remaining data is longer than the expected packet length,
-        # there might be an incomplete packet, so log a warning.
-        if len(self.__tmp_data[tmp_source]) >= byte_length:
-            print("Warning: Incomplete or malformed packet ------------------------------------")
-            self.__tmp_data[tmp_source] = b''
+            if match.start(0) != 0:
+                print(f"skipping {match.start(0)} bytes")
+            # Remove the processed packet from the temporary buffer
+            self.__tmp_data[tmp_source] = self.__tmp_data[tmp_source][match.end():]
 
         return packets
 
+
     def fs_down_callback(self, data):
         # copied from listen_upd()
         if not data:
@@ -255,12 +308,13 @@ def sigint_handler(signal, frame):
 
 def start_comms():
     # start file sync
-    p.start()
-
-
+    # p.start()
+    
     # Start two live comm channels
-    vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL)))
-    vps_thread.start()
-    socket_thread = threading.Thread(target=lambda: telemetry.listen_udp(config.UDP_PORT))
+    #vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL)))
+    #vps_thread.start()
+    #socket_thread = threading.Thread(target=lambda: telemetry.listen_udp(config.UDP_PORT))
+    #socket_thread.start()
+    socket_thread = threading.Thread(target=lambda: telemetry.serial_read())
     socket_thread.start()
 

Backend/core/core_api.py

@@ -1,31 +1,43 @@
 from fastapi import APIRouter, WebSocket
+import serial.tools.list_ports
 from . import comms
+from pydantic import BaseModel
+
 router = APIRouter()
 
-@router.websocket("/single-values")
-async def single_values(websocket: WebSocket):
-    await websocket.accept()
-    
-    try:
-        while True:
-            # wait for client to request more data to be send
-            message_from_client = await websocket.receive_text()
-            
-            if comms.solar_car_connection['udp'] or comms.solar_car_connection['lte']:
-                latest_data = comms.frontend_data
-                latest_data['solar_car_connection'] = True
-                latest_data['udp_status'] = comms.solar_car_connection['udp']
-                latest_data['lte_status'] = comms.solar_car_connection['lte']
-                latest_data['timestamps'] = f'{latest_data["tstamp_hr"]:02d}:{latest_data["tstamp_mn"]:02d}:' \
-                                            f'{latest_data["tstamp_sc"]:02d}.{latest_data["tstamp_ms"]}'
-                format_data = {}
-                for key in latest_data.keys():
-                    format_data[key] = [latest_data[key]]
-                json_data = {'response': format_data}
-                await websocket.send_json(json_data)
-            else:
-                await websocket.send_json({'response': None})
-    except Exception as e:
-        await websocket.send_json({'error': 'Error fetching data'})
-    finally:
-        await websocket.close()
+@router.get("/single-values")
+async def single_values():
+    if comms.solar_car_connection['udp'] or comms.solar_car_connection['lte'] or comms.solar_car_connection['serial']:
+        latest_data = comms.frontend_data
+        latest_data['solar_car_connection'] = True
+        latest_data['udp_status'] = comms.solar_car_connection['udp']
+        latest_data['lte_status'] = comms.solar_car_connection['lte']
+        latest_data['serial_status'] = comms.solar_car_connection['serial']
+        latest_data['timestamps'] = f'{latest_data["tstamp_hr"]:02d}:{latest_data["tstamp_mn"]:02d}:' \
+                                    f'{latest_data["tstamp_sc"]:02d}.{latest_data["tstamp_ms"]}'
+        format_data = {}
+        for key in latest_data.keys():
+            format_data[key] = [latest_data[key]]
+        json_data = {'response': format_data}
+        return json_data
+    return {'response': None}
+
+
+@router.get("/serial-info")
+async def list_serial_ports():
+    """return currently connected device and all available serial device"""
+    ports = serial.tools.list_ports.comports()
+    # Extract the device name from each port object
+    return {'connected_device': {'device': comms.serial_port['device'], 'baud': comms.serial_port['baud']},
+            'all_devices': [port.device for port in sorted(ports, key=lambda port: port.device)]
+            }
+
+class SerialDevice(BaseModel):
+    device: str
+    baud: int
+
+@router.post("/connect-device")
+async def dev_conn(serial_device: SerialDevice):
+    """Connect to serial port, pass in empty device name for disconnect"""
+    comms.serial_port['device'] = serial_device.device
+    comms.serial_port['baud'] = serial_device.baud

Backend/main.py

@@ -11,6 +11,6 @@ async def startup():
     process.start_processes()
 
 if __name__ == '__main__':
-    uvicorn.run(app='main:app', host="0.0.0.0", port=config.HOST_PORT)
+    uvicorn.run(app='main:app', host="0.0.0.0", port=config.HOST_PORT, log_level='critical')
 
 

Backend/setup.py

@@ -9,5 +9,5 @@
     author='Badger Solar Racing Software Team',
     author_email='',
     description='',
-    install_requires=['uvicorn','fastapi','redis', 'requests', 'numpy', 'XlsxWriter', 'pandas', 'aiohttp']
+    install_requires=['uvicorn','fastapi','redis', 'requests', 'numpy', 'XlsxWriter', 'pandas', 'aiohttp', 'pyserial']
 )

Frontend/src/Components/Communication/Communication.js

@@ -38,12 +38,8 @@ export default function Communication(props) {
         <Flex flex='auto' direction='column' overflowY='auto' height='90%'>
             <HeadingCell fontSize='2.2vh' label='Communication'/>
             <Flex flex='inherit' direction='column' pl='2' pt='2' >
-                <CommsLabel
-                    boolean={props.data?.solar_car_connection[0]}
-                    label='Solar Car Connection'
-                />
                 <HStack>
-                    <Text fontSize='2vh' style={{ textIndent: 30 }}>&#160;Packet Delay: </Text>
+                    <Text fontSize='2vh'>&#160;Packet Delay: </Text>
                     <Text fontSize='2vh' backgroundColor={bgColor}>{_getFormattedPacketDelay()}</Text>
                 </HStack>
                 <CommsLabel
@@ -56,6 +52,11 @@ export default function Communication(props) {
                         boolean={props.data?.lte_status[0]}
                         label='LTE'
                 />
+                <CommsLabel
+                        indent={true}
+                        boolean={props.data?.serial_status[0]}
+                        label='Serial'
+                />
                 <CommsLabel
                     boolean={props.data?.mainIO_heartbeat[0]}
                     label='Main IO Heartbeat'

Frontend/src/Components/Dashboard/Dashboard.js

@@ -11,6 +11,7 @@ import GraphContainer from "./GraphContainer";
 import DataRecordingControl from "./DataRecordingControl";
 import dvOptions from "./dataViewOptions";
 import getColor from "../Shared/colors";
+import SerialSelector from "../SerialSelector/SerialSelector";
 import { ROUTES } from "../Shared/misc-constants";
 import FaultsView from "../Faults/FaultsView";
 import fauxQueue from "../Graph/faux-queue.json";
@@ -369,6 +370,18 @@ export default function Dashboard(props) {
           </Select>
           {switchDataView(dataView4)}
         </GridItem>
+        <GridItem
+          minH="min-content"
+          rowStart={4}
+          rowSpan={1}
+          colStart={1}
+          colSpan={2}
+          borderColor={borderCol}
+          borderWidth={1}
+          p={1}
+        >
+          <SerialSelector/>
+        </GridItem>
       </Grid>
       <GraphContainer
         flex="2 2 0"

Frontend/src/Components/SerialSelector/SerialSelector.js

@@ -0,0 +1,81 @@
+import { useEffect, useState } from "react";
+import { Select, Flex, useInterval } from "@chakra-ui/react";
+
+export default function SerialSelector() {
+    const [allDevices, setAllDevices] = useState([]);
+    const [selectedDevice, setSelectedDevice] = useState(""); // State to hold the selected device name
+    const [selectedBaud, setSelectedBaud] = useState(115200);
+
+    const refresh = () => {
+        fetch('/serial-info')
+        .then((response) => {
+            if (response.ok) {
+                return response.json();
+            } else {
+                throw new Error(`Error fetching serial port with code ${response.status}`);
+            }
+        })
+        .then((body) => {
+            setSelectedDevice(body['connected_device']['device']); // Set default device
+            setSelectedBaud(body['connected_device']['baud']);
+            setAllDevices(body['all_devices']);
+        }).catch(error => console.error('Fetch error:', error));
+    };
+
+    useInterval(refresh, 3000);
+
+    useEffect(()=> {
+        fetch("/connect-device", {
+            method: "POST",
+            body: JSON.stringify({
+                device: selectedDevice,
+                baud: selectedBaud
+            }),
+            headers: {
+                "Content-type": "application/json"
+            }
+        });
+
+    }, [selectedBaud, selectedDevice])
+
+    const getSerialPort = () => {
+        return (
+            <Select
+                placeholder='Select option'
+                width={'30%'}
+                padding={2}
+                value={selectedDevice} // Controlled component with selectedDevice as the current value
+                onChange={e => setSelectedDevice(e.target.value)} // Handler to update state on user selection
+            >
+                {allDevices.map(device => (
+                    <option key={device} value={device}>{device}</option>
+                ))}
+            </Select>
+        );
+    };
+
+    const getBaud = () => {
+        const defaultBaud = [4800, 9600, 14400, 19200, 38400, 57600, 115200, 128000, 256000];
+        return (
+            <Select
+                width={'30%'}
+                padding={2}
+                value={selectedBaud}
+                onChange={e => setSelectedBaud(e.target.value)}
+            >
+                {defaultBaud.map(baud => (
+                    <option key={baud} value={baud}>{baud}</option>
+                ))}
+            </Select>
+        )
+    }
+
+    return (
+        <Flex flex="auto" direction="row" alignItems="center" justifyContent="center">
+            <p>Serial Device:</p>
+            {getSerialPort()}
+            <p>Baud:</p>
+            {getBaud()}
+        </Flex>
+    );
+}