add and set alert_end_time metric (#806)

- Add alert_end_time as a prometheus metric. It will be a unix timestamp. The default value should be 0.
- Set alert_end_time when session is created (if one exists and is further in the future than the current alert_end_time)
- Update alert_end_time when session end time is updated

Author: Abigail-Yates

src/murfey/server/api/instrument.py

@@ -246,6 +246,13 @@ async def update_visit_end_time(
     db.add(session_entry)
     db.commit()
 
+    # Update the alert endtime on prometheus
+    alert_end_time = prom.alert_end_time._value.get()
+    if end_time:
+        visit_end_timestamp = end_time.timestamp()
+        if alert_end_time < visit_end_timestamp:
+            prom.alert_end_time.set(visit_end_timestamp)
+
     # Update the multigrid controller
     data = {}
     machine_config = get_machine_config(instrument_name=instrument_name)[

src/murfey/server/api/session_info.py

@@ -10,6 +10,7 @@
 from sqlmodel import select
 
 import murfey.server.api.websocket as ws
+import murfey.server.prometheus as prom
 from murfey.server import _transport_object
 from murfey.server.api import templates
 from murfey.server.api.auth import (
@@ -186,6 +187,13 @@ def create_session(
     db.add(s)
     db.commit()
     sid = s.id
+
+    alert_end_time = prom.alert_end_time._value.get()  # timestamp
+    if visit_end_time.end_time:
+        visit_end_timestamp = visit_end_time.end_time.timestamp()
+        if alert_end_time < visit_end_timestamp:
+            prom.alert_end_time.set(visit_end_timestamp)
+
     return sid
 
 
@@ -300,7 +308,7 @@ def get_silences(instrument_name: MurfeyInstrumentName):
     response = requests.get(f"{alertmanager_url}/api/v2/silences?{query_params}")
     if response.status_code != 200:
         logger.warning(
-            f"Tried to get silences for {sanitise(instrument_name)}, but received status {response.status_code} from alertmanager API"
+            f"Get silences for {sanitise(instrument_name)} received status {response.status_code} from alertmanager API"
         )
     active_silences = []
     for silence in response.json():

src/murfey/server/prometheus.py

@@ -43,3 +43,5 @@
     "Whether the corresponding visit should be monitored or not",
     ["visit"],
 )
+
+alert_end_time = Gauge("alert_end_time", "End time for alerts", [])