refactor: harden monitoring and telemetry handling

- consolidate Docker workload fallback and status handling
- improve UTC timestamp and anomaly-zone processing
- correct telemetry metrics and health-status precedence
- remove dead frontend helpers and expand regression coverage

Author: OliverHennhoefer

backend/services/middleware/tactic/src/off_key_tactic_middleware/api/v1/radar.py

@@ -232,18 +232,19 @@ async def stop_radar_service(
         )
 
     try:
+        lookup_target = container_name if container_name else container_id
         success = await service.stop_radar_service(container_name, container_id)
 
         if not success:
             raise HTTPException(
                 status_code=404,
-                detail=f"RADAR service '{container_name}'"
+                detail=f"RADAR service '{lookup_target}'"
                 f" not found or could not be stopped",
             )
 
         return {
             "status": "stopped",
-            "message": f"RADAR service '{container_name}'"
+            "message": f"RADAR service '{lookup_target}'"
             f" stopped and deleted successfully",
         }
     except HTTPException:

backend/services/middleware/tactic/src/off_key_tactic_middleware/facades/docker.py

@@ -1,13 +1,34 @@
 import asyncio
 import time
 
-import docker.errors
+import docker
 from docker import DockerClient
 from typing import Callable, Any
 from off_key_core.config.logs import logger, log_performance
 from ..config.config import get_tactic_settings
 
 
+_SWARM_FALLBACK_INDICATORS = (
+    "cannot be used with services",
+    "only networks scoped to the swarm can be used",
+    "this node is not a swarm manager",
+    "swarm mode",
+)
+
+
+def _extract_latest_workload_state(
+    tasks: list[object], no_tasks_state: str = "no_tasks"
+) -> str:
+    task_items = [task for task in tasks if isinstance(task, dict)]
+    if not task_items:
+        return no_tasks_state
+    latest = max(task_items, key=lambda task: str(task.get("CreatedAt", "")))
+    status = latest.get("Status")
+    if not isinstance(status, dict):
+        status = {}
+    return str(status.get("State", "unknown")).strip().lower()
+
+
 class AsyncDocker:
     def __init__(self, docker_config=None):
         config = docker_config or get_tactic_settings().config.docker
@@ -52,19 +73,11 @@ def close(self) -> None:
                 logger.warning(f"Error closing Docker client: {e}")
 
 
-def _should_fallback_to_container(exc: Exception) -> bool:
+def should_fallback_to_container(exc: Exception) -> bool:
     if isinstance(exc, docker.errors.APIError) and exc.status_code in (400, 406, 503):
         return True
     text = str(exc).lower()
-    return any(
-        indicator in text
-        for indicator in (
-            "cannot be used with services",
-            "only networks scoped to the swarm can be used",
-            "this node is not a swarm manager",
-            "swarm mode is not active",
-        )
-    )
+    return any(indicator in text for indicator in _SWARM_FALLBACK_INDICATORS)
 
 
 async def get_workload_docker_status(
@@ -73,9 +86,9 @@ async def get_workload_docker_status(
     """Return the running status of a Docker workload by ID.
 
     Tries Swarm services first (reads task state), then falls back to plain
-    containers (reads container status). Both callers — RadarOrchestrationService
-    and RadarStatusReconciliationService — delegate here to avoid duplicating the
-    service→container probe-and-fallback logic.
+    containers (reads container status). Both callers -- RadarOrchestrationService
+    and RadarStatusReconciliationService -- delegate here to avoid duplicating the
+    service->container probe-and-fallback logic.
 
     Returns:
         "running" | task state | "no_tasks" | "not_found" | "no_container_id"
@@ -89,14 +102,11 @@ async def get_workload_docker_status(
                 async_docker.client.services.get, container_id
             )
             tasks = await async_docker.run(docker_service.tasks)
-            if tasks:
-                latest = max(tasks, key=lambda t: t.get("CreatedAt", ""))
-                return latest.get("Status", {}).get("State", "unknown")
-            return "no_tasks"
+            return _extract_latest_workload_state(tasks)
         except docker.errors.NotFound:
             pass
         except Exception as exc:
-            if not _should_fallback_to_container(exc):
+            if not should_fallback_to_container(exc):
                 raise
             logger.debug(
                 "Skipping Swarm service status lookup for workload %s: %s",
@@ -114,3 +124,32 @@ async def get_workload_docker_status(
     except Exception as exc:
         logger.debug("Error checking Docker status for %s: %s", container_id, exc)
         return "error"
+
+
+async def with_workload_fallback(
+    async_docker: AsyncDocker,
+    container_id: str,
+    on_service: Callable[[Any], Any],
+    on_container: Callable[[Any], Any],
+) -> Any:
+    """Resolve a Docker workload as swarm service first, then fallback to container."""
+    try:
+        docker_service = await async_docker.run(
+            async_docker.client.services.get, container_id
+        )
+        return await async_docker.run(on_service, docker_service)
+    except docker.errors.NotFound:
+        pass
+    except Exception as exc:
+        if not should_fallback_to_container(exc):
+            raise
+        logger.debug(
+            "Skipping Swarm service operation for workload %s: %s",
+            container_id,
+            exc,
+        )
+
+    docker_container = await async_docker.run(
+        async_docker.client.containers.get, container_id
+    )
+    return await async_docker.run(on_container, docker_container)