Add heartbeat timeout warning (#1164)

* Add heartbeat timeout warning

* Address copilot review comments

Author: 1Blademaster

gcs/data/default_settings.json

@@ -79,6 +79,18 @@
       "display": "Battery Alert",
       "suffix": "%",
       "group": "Battery"
+    },
+    "heartbeatTimeoutSeconds": {
+      "description": "You'll be notified when heartbeats stop arriving from the aircraft for this long.",
+      "default": 10,
+      "type": "number",
+      "range": [
+        1,
+        3600
+      ],
+      "display": "Heartbeat Timeout",
+      "suffix": "s",
+      "group": "Connection"
     }
   },
   "Video": {
@@ -203,4 +215,4 @@
       "group": "Data stream rates"
     }
   }
-}
+}

gcs/src/components/dashboard/alerts/alert.jsx

@@ -22,10 +22,14 @@ export default function AlertSection() {
           <Alert
             variant="outline"
             color={SeverityColor[alert.severity]}
-            withCloseButton
+            withCloseButton={alert.dismissable !== false}
             title={alert.category}
             icon={<IconAlertTriangle />}
-            onClose={() => dismissAlert(alert.category, true)}
+            onClose={
+              alert.dismissable === false
+                ? undefined
+                : () => dismissAlert(alert.category, true)
+            }
           >
             {alert.jsx}
           </Alert>

gcs/src/components/dashboard/alerts/alertConstants.js

@@ -1,6 +1,7 @@
 export const AlertCategory = {
   Altitude: "Altitude",
   Battery: "Battery",
+  Heartbeat: "Heartbeat",
   Speed: "Speed",
 }
 

gcs/src/components/dashboard/alerts/alertProvider.jsx

@@ -31,6 +31,11 @@ export default function AlertProvider({ children }) {
   function dismissAlert(category, manual) {
     setAlerts((prevAlerts) => {
       const alert = prevAlerts.find((a) => a.category === category)
+      if (!alert) return prevAlerts
+
+      if (manual && alert.dismissable === false) {
+        return prevAlerts
+      }
 
       if (manual) {
         dismissedAlerts.current.set(category, alert.severity)

gcs/src/components/dashboard/statusBar.jsx

@@ -13,7 +13,12 @@ import { IconClock, IconNetwork, IconNetworkOff } from "@tabler/icons-react"
 
 // Redux
 import { useSelector } from "react-redux"
-import { selectAlt, selectBatteryData } from "../../redux/slices/droneInfoSlice"
+import {
+  selectAlt,
+  selectBatteryData,
+  selectHeartbeatLastReceivedAt,
+} from "../../redux/slices/droneInfoSlice"
+import { selectConnectedToDrone } from "../../redux/slices/droneConnectionSlice"
 import { selectIsConnectedToSocket } from "../../redux/slices/socketSlice"
 
 // Helper imports
@@ -36,9 +41,11 @@ export function StatusSection({ icon, value, tooltip }) {
 
 export default function StatusBar(props) {
   const isConnectedToSocket = useSelector(selectIsConnectedToSocket)
+  const isConnectedToDrone = useSelector(selectConnectedToDrone)
   const [time, setTime] = useState(moment())
   const batteryData = useSelector(selectBatteryData)
   const alt = useSelector(selectAlt)
+  const heartbeatLastReceivedAt = useSelector(selectHeartbeatLastReceivedAt)
 
   // Update clock every second
   useEffect(() => {
@@ -50,6 +57,7 @@ export default function StatusBar(props) {
   const { getSetting } = useSettings()
   const { dispatchAlert, dismissAlert } = useAlerts()
   const highestAltitudeRef = useRef(0)
+  const heartbeatAlertActiveRef = useRef(false)
 
   useEffect(() => {
     const maxAltitude = getSetting("Dashboard.maxAltitudeAlert")
@@ -126,6 +134,69 @@ export default function StatusBar(props) {
     })
   }, [batteryData])
 
+  useEffect(() => {
+    const timeoutSeconds = Number(
+      getSetting("Dashboard.heartbeatTimeoutSeconds") ?? 10,
+    )
+    const timeoutMs = Number.isFinite(timeoutSeconds)
+      ? timeoutSeconds * 1000
+      : 10000
+
+    const checkHeartbeatTimeout = () => {
+      if (!isConnectedToDrone) {
+        if (heartbeatAlertActiveRef.current) {
+          dismissAlert(AlertCategory.Heartbeat)
+          heartbeatAlertActiveRef.current = false
+        }
+        return
+      }
+
+      if (heartbeatLastReceivedAt <= 0) {
+        if (heartbeatAlertActiveRef.current) {
+          dismissAlert(AlertCategory.Heartbeat)
+          heartbeatAlertActiveRef.current = false
+        }
+        return
+      }
+
+      const elapsedSeconds = Math.max(
+        0,
+        Math.floor((Date.now() - heartbeatLastReceivedAt) / 1000),
+      )
+      const isHeartbeatStale = elapsedSeconds * 1000 > timeoutMs
+
+      if (isHeartbeatStale) {
+        dispatchAlert({
+          category: AlertCategory.Heartbeat,
+          severity: AlertSeverity.Red,
+          dismissable: false,
+          jsx: (
+            <>
+              Caution! It has been {elapsedSeconds}s since the last heartbeat.
+            </>
+          ),
+        })
+        heartbeatAlertActiveRef.current = true
+        return
+      }
+
+      if (heartbeatAlertActiveRef.current) {
+        dismissAlert(AlertCategory.Heartbeat)
+        heartbeatAlertActiveRef.current = false
+      }
+    }
+
+    checkHeartbeatTimeout()
+    const id = setInterval(checkHeartbeatTimeout, 1000)
+    return () => clearInterval(id)
+  }, [
+    dismissAlert,
+    dispatchAlert,
+    getSetting,
+    heartbeatLastReceivedAt,
+    isConnectedToDrone,
+  ])
+
   return (
     <div className={`${props.className} flex flex-col items-end`}>
       <div

gcs/src/helpers/settingsProvider.jsx

@@ -1,4 +1,4 @@
-import { createContext, useEffect, useState } from "react"
+import { createContext, useCallback, useEffect, useMemo, useState } from "react"
 
 import { getSettingFromSettings, setSettingInSettings } from "./settings"
 
@@ -42,35 +42,46 @@ export const SettingsProvider = ({ children }) => {
     return () => window.ipcRenderer.removeAllListeners("settings:open")
   }, [open])
 
-  const setSetting = (setting, value) => {
-    if (settings === null) return
+  const setSetting = useCallback(
+    (setting, value) => {
+      if (settings === null) return
 
-    const newSettings = {
-      version: settings.version,
-      settings: setSettingInSettings(setting, value, settings.settings),
-    }
+      const newSettings = {
+        version: settings.version,
+        settings: setSettingInSettings(setting, value, settings.settings),
+      }
 
-    setSettings(newSettings)
-    window.ipcRenderer.invoke("settings:save-settings", newSettings)
-  }
+      setSettings(newSettings)
+      window.ipcRenderer.invoke("settings:save-settings", newSettings)
+    },
+    [settings],
+  )
 
-  const getSetting = (setting) => {
-    const userSetting = getSettingFromSettings(setting, settings.settings)
-    const defaultSetting = getSettingFromSettings(setting, DefaultSettings)
+  const getSetting = useCallback(
+    (setting) => {
+      if (settings === null) return null
 
-    if (userSetting !== null) return userSetting
+      const userSetting = getSettingFromSettings(setting, settings.settings)
+      const defaultSetting = getSettingFromSettings(setting, DefaultSettings)
 
-    if (defaultSetting === null || defaultSetting === undefined) {
-      return null
-    }
+      if (userSetting !== null) return userSetting
 
-    return defaultSetting.default
-  }
+      if (defaultSetting === null || defaultSetting === undefined) {
+        return null
+      }
+
+      return defaultSetting.default
+    },
+    [settings],
+  )
+
+  const contextValue = useMemo(
+    () => ({ getSetting, setSetting, settings, opened, open, close }),
+    [getSetting, setSetting, settings, opened, open, close],
+  )
 
   return (
-    <SettingsContext.Provider
-      value={{ getSetting, setSetting, settings, opened, open, close }}
-    >
+    <SettingsContext.Provider value={contextValue}>
       {settings !== null ? children : <></>}
     </SettingsContext.Provider>
   )

gcs/src/redux/middleware/heartbeatMonitorMiddleware.js

@@ -0,0 +1,33 @@
+import {
+  setHeartbeatData,
+  resetHeartbeatMonitor,
+  setHeartbeatMonitorLastReceivedAt,
+} from "../slices/droneInfoSlice"
+import { setConnected } from "../slices/droneConnectionSlice"
+import { socketDisconnected } from "../slices/socketSlice"
+
+const heartbeatMonitorMiddleware = (store) => {
+  return (next) => (action) => {
+    const result = next(action)
+
+    if (setConnected.match(action)) {
+      if (action.payload === true) {
+        store.dispatch(setHeartbeatMonitorLastReceivedAt(Date.now()))
+      } else if (action.payload === false) {
+        store.dispatch(resetHeartbeatMonitor())
+      }
+    }
+
+    if (socketDisconnected.match(action)) {
+      store.dispatch(resetHeartbeatMonitor())
+    }
+
+    if (setHeartbeatData.match(action)) {
+      store.dispatch(setHeartbeatMonitorLastReceivedAt(Date.now()))
+    }
+
+    return result
+  }
+}
+
+export default heartbeatMonitorMiddleware

gcs/src/redux/slices/droneInfoSlice.js

@@ -66,6 +66,9 @@ const droneInfoSlice = createSlice({
       customMode: 0,
       systemStatus: 0,
     },
+    heartbeatMonitor: {
+      lastReceivedAt: 0,
+    },
     onboardControlSensorsEnabled: 0,
     onboardControlSensorsHealth: 0,
     gpsRawIntData: {
@@ -156,6 +159,14 @@ const droneInfoSlice = createSlice({
       state.heartbeatData.customMode = action.payload.custom_mode
       state.heartbeatData.systemStatus = action.payload.system_status
     },
+    setHeartbeatMonitorLastReceivedAt: (state, action) => {
+      if (action.payload !== state.heartbeatMonitor.lastReceivedAt) {
+        state.heartbeatMonitor.lastReceivedAt = action.payload
+      }
+    },
+    resetHeartbeatMonitor: (state) => {
+      state.heartbeatMonitor.lastReceivedAt = 0
+    },
     setBatteryData: (state, action) => {
       const battery = state.batteryData.filter(
         (battery) => battery.id == action.payload.id,
@@ -381,6 +392,8 @@ const droneInfoSlice = createSlice({
     selectNavController: (state) => state.navControllerData,
     selectDesiredBearing: (state) => state.navControllerData.navBearing,
     selectHeartbeat: (state) => state.heartbeatData,
+    selectHeartbeatLastReceivedAt: (state) =>
+      state.heartbeatMonitor.lastReceivedAt,
     selectIsArmed: (state) => state.isArmed,
     selectIsFlying: (state) => state.isFlying,
     selectNotificationSound: (state) => state.notificationSound,
@@ -425,6 +438,8 @@ const droneInfoSlice = createSlice({
 export const {
   setFlightSwVersion,
   setHeartbeatData,
+  setHeartbeatMonitorLastReceivedAt,
+  resetHeartbeatMonitor,
   soundPlayed,
   changeSelectedDisplayTelemetry,
   setSelectedDisplayTelemetry,
@@ -593,6 +608,7 @@ export const {
   selectNavController,
   selectDesiredBearing,
   selectHeartbeat,
+  selectHeartbeatLastReceivedAt,
   selectIsArmed,
   selectIsFlying,
   selectReadyToArm,

gcs/src/redux/store.js

@@ -1,5 +1,6 @@
 import { combineSlices, configureStore } from "@reduxjs/toolkit"
 import { defaultDataMessages } from "../helpers/dashboardDefaultDataMessages"
+import heartbeatMonitorMiddleware from "./middleware/heartbeatMonitorMiddleware"
 import armedMiddleware from "./middleware/armedMiddleware"
 import socketMiddleware from "./middleware/socketMiddleware"
 import applicationSlice from "./slices/applicationSlice"
@@ -54,7 +55,7 @@ export const store = configureStore({
     return getDefaultMiddleware({
       immutableCheck: false,
       serializableCheck: false,
-    }).concat([socketMiddleware, armedMiddleware])
+    }).concat([socketMiddleware, heartbeatMonitorMiddleware, armedMiddleware])
   },
 })
 

radio/app/drone.py

@@ -761,8 +761,6 @@ def checkForMessages(self) -> None:
                 break
             except (serial.serialutil.SerialException, ConnectionAbortedError):
                 self.logger.error("Autopilot disconnected", exc_info=True)
-                if self.droneDisconnectCb:
-                    self.droneDisconnectCb()
                 self.close()
                 break
             except Exception as e:
@@ -1282,6 +1280,9 @@ def close(self) -> None:
         self.logger.info(f"Cleaning up resources for drone at {self}")
         self.clearAllMessageListeners()
 
+        if self.droneDisconnectCb:
+            self.droneDisconnectCb()
+
         self.is_active.clear()
 
         if getattr(self, "master", None) is not None: