Merge pull request #4 from Jeremiah-Sakuda/phase-4-recovery-verification

Phase 4: Recovery verification with real health polling

Author: Jeremiah-Sakuda

src/autosre/config.py

@@ -31,6 +31,8 @@ class Settings(BaseSettings):
     # Demo / operations dashboard
     operations_dashboard_url: str = "http://localhost:3000"
     incident_source: str = "simulated"
+    # Health/metrics URL for recovery verification (default: dashboard + /api/health)
+    metrics_url: str = ""
 
     # UI automation (Nova Act): set False to use real browser; True for stub (CI/demo without browser)
     ui_stub: bool = True

src/autosre/recovery_verification/monitor.py

@@ -1,25 +1,76 @@
 """Verifies service health after remediation actions."""
 
+import logging
 import time
 
+import httpx
+
 from autosre.models import RecoveryStatus
 
+logger = logging.getLogger(__name__)
+
+DEFAULT_POLL_INTERVAL = 3.0
+STUB_RECOVERY_SECONDS = 92.0
+
 
 class RecoveryMonitor:
     """
-    Checks metrics (latency, error rate) to confirm recovery.
+    Checks metrics (health endpoint) to confirm recovery.
 
-    Stub: waits briefly then returns RECOVERED for deterministic demo.
+    When metrics_url is set: polls GET metrics_url until response has
+    status == "healthy" or timeout. Tracks recovery time from action_start_time.
+    When metrics_url is empty: stub behavior (short sleep, RECOVERED) for demo/CI.
     """
 
+    def __init__(self, metrics_url: str | None = None) -> None:
+        self._metrics_url = (metrics_url or "").strip()
+        self._last_recovery_seconds: float = 0.0
+
     def verify(
-        self, incident_id: str, service_name: str, timeout_seconds: float = 120
+        self,
+        incident_id: str,
+        service_name: str,
+        timeout_seconds: float = 120,
+        action_start_time: float | None = None,
     ) -> RecoveryStatus:
-        """Poll until recovery signals or timeout. Returns recovery status."""
-        # Stub: short wait then recovered
-        time.sleep(1)
-        return RecoveryStatus.RECOVERED
+        """Poll until recovery (status == healthy) or timeout. Returns recovery status."""
+        if not self._metrics_url:
+            time.sleep(1)
+            self._last_recovery_seconds = STUB_RECOVERY_SECONDS
+            return RecoveryStatus.RECOVERED
+
+        start = action_start_time if action_start_time is not None else time.monotonic()
+        deadline = start + timeout_seconds
+        poll_interval = DEFAULT_POLL_INTERVAL
+
+        while time.monotonic() < deadline:
+            try:
+                with httpx.Client(timeout=5.0) as client:
+                    r = client.get(self._metrics_url)
+                    r.raise_for_status()
+                    data = r.json()
+                    if data.get("status") == "healthy":
+                        self._last_recovery_seconds = time.monotonic() - start
+                        logger.info(
+                            "Recovery verified",
+                            extra={
+                                "incident_id": incident_id,
+                                "service_name": service_name,
+                                "recovery_seconds": self._last_recovery_seconds,
+                            },
+                        )
+                        return RecoveryStatus.RECOVERED
+            except Exception as e:
+                logger.debug("Health poll failed: %s", e)
+            time.sleep(poll_interval)
+
+        self._last_recovery_seconds = timeout_seconds
+        logger.warning(
+            "Recovery timeout",
+            extra={"incident_id": incident_id, "service_name": service_name},
+        )
+        return RecoveryStatus.NOT_RECOVERED
 
     def get_recovery_time_seconds(self) -> float:
-        """Return elapsed time from action to recovery (stub)."""
-        return 92.0
+        """Return elapsed time from action to recovery (from last verify run)."""
+        return self._last_recovery_seconds

src/autosre/workflow.py

@@ -5,6 +5,8 @@
     → Health verification → Slack post-mortem
 """
 
+import time
+
 from autosre.config import get_settings
 from autosre.incident_detection import get_incident_stream
 from autosre.log_storage import LogStore
@@ -58,7 +60,11 @@ def run_once(incident_type: IncidentType | None = None) -> bool:
         return False
 
     # 5. Recovery verification
-    status = monitor.verify(incident.incident_id, incident.service_name)
+    status = monitor.verify(
+        incident.incident_id,
+        incident.service_name,
+        action_start_time=action_start_time,
+    )
     recovery_seconds = monitor.get_recovery_time_seconds()
 
     # 6. Post-mortem to Slack

tests/test_recovery_verification.py

@@ -0,0 +1,107 @@
+"""Tests for recovery verification (Phase 4)."""
+
+import time
+from unittest.mock import MagicMock, patch
+
+from autosre.models import RecoveryStatus
+from autosre.recovery_verification.monitor import STUB_RECOVERY_SECONDS, RecoveryMonitor
+
+
+def test_monitor_stub_when_no_metrics_url():
+    """When metrics_url is empty, returns RECOVERED after short sleep (stub)."""
+    monitor = RecoveryMonitor(metrics_url="")
+    status = monitor.verify("inc-1", "checkout", timeout_seconds=120)
+    assert status == RecoveryStatus.RECOVERED
+    assert monitor.get_recovery_time_seconds() == STUB_RECOVERY_SECONDS
+
+
+def test_monitor_stub_with_none_metrics_url():
+    """When metrics_url is None, uses stub behavior."""
+    monitor = RecoveryMonitor(metrics_url=None)
+    status = monitor.verify("inc-1", "checkout")
+    assert status == RecoveryStatus.RECOVERED
+    assert monitor.get_recovery_time_seconds() == STUB_RECOVERY_SECONDS
+
+
+@patch("autosre.recovery_verification.monitor.time.sleep")
+@patch("autosre.recovery_verification.monitor.httpx.Client")
+def test_monitor_recovered_when_healthy(mock_client_class, mock_sleep):
+    """When GET returns status healthy, returns RECOVERED and sets recovery time."""
+    mock_response = MagicMock()
+    mock_response.raise_for_status = MagicMock()
+    mock_response.json.return_value = {"status": "healthy"}
+    mock_client = MagicMock()
+    mock_client.get.return_value = mock_response
+    mock_client_class.return_value.__enter__.return_value = mock_client
+
+    monitor = RecoveryMonitor(metrics_url="http://localhost:3000/api/health")
+    action_start = time.monotonic()
+    status = monitor.verify(
+        "inc-1",
+        "checkout",
+        timeout_seconds=30,
+        action_start_time=action_start,
+    )
+    assert status == RecoveryStatus.RECOVERED
+    recovery_sec = monitor.get_recovery_time_seconds()
+    assert recovery_sec >= 0
+    assert recovery_sec < 10
+
+
+@patch("autosre.recovery_verification.monitor.httpx.Client")
+@patch("autosre.recovery_verification.monitor.time.sleep")
+def test_monitor_polls_until_healthy(mock_sleep, mock_client_class):
+    """Polls until response is healthy; recovery time reflects delay."""
+    def mk_resp(status: str):
+        r = MagicMock()
+        r.raise_for_status = MagicMock()
+        r.json.return_value = {"status": status}
+        return r
+
+    mock_client = mock_client_class.return_value.__enter__.return_value
+    mock_client.get.side_effect = [
+        mk_resp("degraded"),
+        mk_resp("degraded"),
+        mk_resp("healthy"),
+    ]
+
+    monitor = RecoveryMonitor(metrics_url="http://localhost:3000/api/health")
+    action_start = time.monotonic()
+    status = monitor.verify(
+        "inc-1",
+        "checkout",
+        timeout_seconds=30,
+        action_start_time=action_start,
+    )
+    assert status == RecoveryStatus.RECOVERED
+    assert mock_client.get.call_count == 3
+    assert mock_sleep.call_count >= 2
+
+
+@patch("autosre.recovery_verification.monitor.httpx.Client")
+@patch("autosre.recovery_verification.monitor.time.monotonic")
+@patch("autosre.recovery_verification.monitor.time.sleep")
+def test_monitor_timeout_returns_not_recovered(mock_sleep, mock_monotonic, mock_client_class):
+    """When timeout is reached without healthy, returns NOT_RECOVERED."""
+    start = 1000.0
+    deadline = start + 0.1
+    # First call: loop condition; then in loop get(); sleep(); loop condition again with time past deadline
+    mock_monotonic.side_effect = [start, start + 0.05, deadline + 0.01]
+    def mk_resp(status: str):
+        r = MagicMock()
+        r.raise_for_status = MagicMock()
+        r.json.return_value = {"status": status}
+        return r
+
+    mock_client = mock_client_class.return_value.__enter__.return_value
+    mock_client.get.return_value = mk_resp("degraded")
+
+    monitor = RecoveryMonitor(metrics_url="http://localhost:3000/api/health")
+    status = monitor.verify(
+        "inc-1",
+        "checkout",
+        timeout_seconds=0.1,
+        action_start_time=start,
+    )
+    assert status == RecoveryStatus.NOT_RECOVERED
+    assert monitor.get_recovery_time_seconds() == 0.1

tests/test_workflow.py

@@ -1,10 +1,19 @@
 """Smoke test for the full workflow (stub components)."""
 
-from autosre.models import IncidentType
+from unittest.mock import MagicMock, patch
+
+from autosre.models import IncidentType, RecoveryStatus
 from autosre.workflow import run_once
 
 
-def test_run_once_latency_spike():
+@patch("autosre.workflow.RecoveryMonitor")
+def test_run_once_latency_spike(mock_monitor_class):
     """One full cycle with latency_spike incident should complete."""
+    mock_monitor = MagicMock()
+    mock_monitor.verify.return_value = RecoveryStatus.RECOVERED
+    mock_monitor.get_recovery_time_seconds.return_value = 92.0
+    mock_monitor_class.return_value = mock_monitor
+
     result = run_once(incident_type=IncidentType.LATENCY_SPIKE)
     assert result is True
+    mock_monitor.verify.assert_called_once()