feat(monitoring): temperature history + temperature alert metric (#601)

Live per-host temperature already shipped via the lm-sensors Sensors panel.
This adds the two pieces #601 asked for on top:

History / chart over time:
- metrics.py collector now records each online node hottest lm-sensors temp
  into the persisted 5-min metrics_history snapshot. SSH fan-out uses the
  SSH-aware run_per_node (max_concurrent=8/cluster) so we never open 30+
  simultaneous sessions; a 1h per-node backoff skips hosts without lm-sensors,
  and the per-manager temp caches are pruned to known nodes.
- new GET /clusters/<id>/nodes/<node>/temperature-history (off-hub, cached).
- node Sensors panel renders a temperature-over-time sparkline + min/max/avg.

Alerting:
- new "temperature" alert metric for node (this node) and cluster (hottest
  node) targets, read from a manager temp-cache the 60s alert loop never SSHes.
  Alerts render in °C (not %); auto-severity uses absolute thresholds
  (>=85 crit, >=75 warn). Alert form gains the metric + a dynamic °C/% unit.
- i18n for all 7 languages.

Author: MrMasterbay

pegaprox/api/nodes.py

@@ -194,6 +194,35 @@ def get_node_sensors_api(cluster_id, node):
     return jsonify(result)
 
 
+# #601 — per-node hottest-temperature history for the chart-over-time ask. Reads the
+# background collector's persisted 5-min metrics_history snapshots (off-hub, cached);
+# no live SSH here, so it stays cheap even at fleet scale.
+@bp.route('/api/clusters/<cluster_id>/nodes/<node>/temperature-history', methods=['GET'])
+@require_auth(perms=['node.view'])
+def get_node_temperature_history_api(cluster_id, node):
+    ok, err = check_cluster_access(cluster_id)
+    if not ok: return err
+    bad, code = _reject_bad_node(node)
+    if bad is not None: return bad, code
+    from pegaprox.background.metrics import load_metrics_history
+    series = []
+    try:
+        hist = load_metrics_history() or {}
+        for snap in (hist.get('snapshots') or []):
+            try:
+                nodes = ((snap.get('clusters') or {}).get(cluster_id) or {}).get('nodes') or {}
+                temp = (nodes.get(node) or {}).get('temp')
+                ts = snap.get('timestamp')
+                if temp is not None and ts:
+                    series.append({'ts': ts, 'temp': temp})
+            except Exception:
+                continue
+    except Exception as e:
+        return jsonify({'error': f'history read failed: {e}'}), 500
+    series.sort(key=lambda x: x['ts'])
+    return jsonify({'series': series, 'unit': '°C', 'count': len(series)})
+
+
 @bp.route('/api/clusters/<cluster_id>/nodes/<node>/network/<iface>', methods=['PUT'])
 @require_auth(perms=['node.network'])
 def update_node_network_api(cluster_id, node, iface):

pegaprox/background/alerts.py

@@ -203,6 +203,16 @@ def check_and_send_alerts():
                         total = sum(n.get('disk_total', 0) for n in online)
                         if total > 0:
                             current_value = used / total * 100
+                    elif metric == 'temperature':
+                        # #601 — hottest cached node temperature (°C) across the cluster.
+                        # Populated by the 5-min metrics collector; we never SSH here.
+                        temps = []
+                        for nname in list(getattr(manager, '_node_temp_cache', {}) or {}):
+                            tv = manager.get_cached_node_temp(nname)
+                            if tv is not None:
+                                temps.append(tv)
+                        if temps:
+                            current_value = max(temps)
                     elif metric in ('backup_sla_breached_pct', 'backup_sla_compliance_pct'):
                         # MK May 2026 — Backup SLA-aware alerts. Run the same
                         # eval as the /backup-sla endpoint and feed the % into
@@ -323,6 +333,9 @@ def _scan_node(nd):
                         rootfs = node_summary.get('rootfs', {}) or {}
                         if rootfs.get('total', 0) > 0:
                             current_value = (rootfs.get('used', 0) / rootfs.get('total', 1)) * 100
+                    elif metric == 'temperature':
+                        # #601 — cached hottest-sensor temp (°C) for this node.
+                        current_value = manager.get_cached_node_temp(target_id)
 
                 elif target_type == 'vm':
                     # MK: was `manager.get_resources()` which doesn't exist; the
@@ -364,8 +377,11 @@ def _scan_node(nd):
         elif operator == '<=' and current_value <= threshold:
             triggered = True
 
+        # #601 — temperature is an absolute °C reading, every other metric is a %.
+        unit = '°C' if metric == 'temperature' else '%'
+
         if not triggered:
-            _record_eval(alert_id, reason=f'below threshold ({metric}={current_value:.1f}% {operator} {threshold}% → false)',
+            _record_eval(alert_id, reason=f'below threshold ({metric}={current_value:.1f}{unit} {operator} {threshold}{unit} → false)',
                          cluster_id=cluster_id, metric=metric, current_value=round(current_value, 1),
                          threshold=threshold, operator=operator, triggered=False)
 
@@ -377,8 +393,8 @@ def _scan_node(nd):
 Alert: {alert_name}
 Target: {target_type.capitalize()} - {target_name}
 Metric: {metric.upper()}
-Condition: {metric} {operator} {threshold}%
-Current Value: {current_value:.1f}%
+Condition: {metric} {operator} {threshold}{unit}
+Current Value: {current_value:.1f}{unit}
 Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
 Cluster: {cluster_id}
 
@@ -394,8 +410,8 @@ def _scan_node(nd):
 <table style="border-collapse: collapse; width: 100%; max-width: 500px;">
 <tr><td style="padding: 8px; border: 1px solid #ddd;"><strong>Target</strong></td><td style="padding: 8px; border: 1px solid #ddd;">{_e(str(target_type).capitalize())} - {_e(str(target_name))}</td></tr>
 <tr><td style="padding: 8px; border: 1px solid #ddd;"><strong>Metric</strong></td><td style="padding: 8px; border: 1px solid #ddd;">{_e(str(metric).upper())}</td></tr>
-<tr><td style="padding: 8px; border: 1px solid #ddd;"><strong>Condition</strong></td><td style="padding: 8px; border: 1px solid #ddd;">{_e(str(metric))} {_e(str(operator))} {threshold}%</td></tr>
-<tr style="background-color: #fee2e2;"><td style="padding: 8px; border: 1px solid #ddd;"><strong>Current Value</strong></td><td style="padding: 8px; border: 1px solid #ddd;"><strong>{current_value:.1f}%</strong></td></tr>
+<tr><td style="padding: 8px; border: 1px solid #ddd;"><strong>Condition</strong></td><td style="padding: 8px; border: 1px solid #ddd;">{_e(str(metric))} {_e(str(operator))} {threshold}{unit}</td></tr>
+<tr style="background-color: #fee2e2;"><td style="padding: 8px; border: 1px solid #ddd;"><strong>Current Value</strong></td><td style="padding: 8px; border: 1px solid #ddd;"><strong>{current_value:.1f}{unit}</strong></td></tr>
 <tr><td style="padding: 8px; border: 1px solid #ddd;"><strong>Time</strong></td><td style="padding: 8px; border: 1px solid #ddd;">{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</td></tr>
 </table>
 <p style="color: #666; font-size: 12px; margin-top: 20px;">This is an automated alert from PegaProx.</p>
@@ -437,7 +453,13 @@ def _scan_node(nd):
 
             # NS #501: a rule may pin an explicit severity; otherwise derive it.
             _rule_sev = alert.get('severity')
-            severity = _rule_sev if _rule_sev and _rule_sev != 'auto' else ('critical' if current_value > 90 else 'warning' if current_value > 70 else 'info')
+            if _rule_sev and _rule_sev != 'auto':
+                severity = _rule_sev
+            elif metric == 'temperature':
+                # #601 — absolute °C thresholds for auto-severity (not the %-based ladder)
+                severity = 'critical' if current_value >= 85 else 'warning' if current_value >= 75 else 'info'
+            else:
+                severity = 'critical' if current_value > 90 else 'warning' if current_value > 70 else 'info'
             alert_data = {
                 'alert_name': alert_name,
                 'metric': metric,
@@ -449,7 +471,7 @@ def _scan_node(nd):
                 'cluster_id': cluster_id,
                 'severity': severity,
                 'timestamp': datetime.now().isoformat(),
-                'message': f"{target_type.capitalize()} {target_name}: {metric} is {current_value:.1f}% (threshold: {operator} {threshold}%)",
+                'message': f"{target_type.capitalize()} {target_name}: {metric} is {current_value:.1f}{unit} (threshold: {operator} {threshold}{unit})",
             }
             if _notification_handlers:
                 for handler in _notification_handlers:

pegaprox/background/metrics.py

@@ -16,6 +16,38 @@
 
 from pegaprox.globals import cluster_managers
 from pegaprox.core.db import get_db
+from pegaprox.utils.concurrent import run_per_node  # #601: SSH-aware bounded fan-out for per-node temp reads
+
+
+def _node_hottest_temp(mgr, node):
+    """#601 — SSH lm-sensors → hottest temperature reading (°C) for one node, or None.
+
+    Honours a per-node backoff so installs WITHOUT lm-sensors/SSH (or non-PVE hosts)
+    aren't re-probed every 5-min cycle — an error parks the node for ~1h. Nodes that
+    do report temps clear their backoff, so a freshly-installed lm-sensors is picked
+    up on the next successful probe.
+    """
+    import time as _t
+    backoff = getattr(mgr, '_node_temp_probe_backoff', None)
+    if backoff is None:
+        backoff = mgr._node_temp_probe_backoff = {}
+    until = backoff.get(node, 0)
+    if until and _t.time() < until:
+        return None
+    try:
+        res = mgr.get_node_sensors(node)
+    except Exception:
+        res = {'error': 'exception'}
+    if not isinstance(res, dict) or res.get('error'):
+        backoff[node] = _t.time() + 3600  # no sensors/SSH here → don't storm SSH
+        return None
+    temps = [s.get('value') for s in (res.get('sensors') or [])
+             if s.get('kind') == 'temp' and isinstance(s.get('value'), (int, float))]
+    backoff.pop(node, None)  # works here — clear any prior backoff
+    if not temps:
+        return None
+    return round(max(temps), 1)
+
 
 def load_metrics_history():
     """Load historical metrics from SQLite database.
@@ -173,7 +205,51 @@ def collect_metrics_snapshot():
                 cluster_data['totals']['cpu_used'] += node_data.get('cpu', 0) * node_data.get('maxcpu', 0)
                 cluster_data['totals']['mem_total'] += node_data.get('maxmem', 0)
                 cluster_data['totals']['mem_used'] += node_data.get('mem', 0)
-            
+
+            # #601 — per-node hottest temperature (°C) for the history chart + the
+            # temperature alert metric. lm-sensors is SSH-side, so fan out over ALL
+            # online nodes with the SSH-AWARE primitive: run_per_node caps concurrency
+            # at 8/cluster so we never open 30+ simultaneous SSH sessions (which trips
+            # AccountLockFailures on hardened nodes). Real PVE (corosync) clusters are
+            # <=~32 nodes, so 90s comfortably covers a whole cluster each cycle. Runs on
+            # the 5-min collector greenlet, off the broadcast hot-path. Writes both the
+            # snapshot (→ persisted history) and the manager temp-cache (→ read by the
+            # 60s alert loop, which must not SSH).
+            try:
+                online_node_names = list(cluster_data['nodes'].keys())
+                if online_node_names:
+                    node_calls = {name: (lambda nm: _node_hottest_temp(mgr, nm))
+                                  for name in online_node_names}
+                    temp_by_node = run_per_node(node_calls, max_concurrent=8, timeout=90)
+                    now_ts = time.time()
+                    tcache = getattr(mgr, '_node_temp_cache', None)
+                    tlock = getattr(mgr, '_node_temp_lock', None)
+                    got = 0
+                    for nm, temp_c in (temp_by_node or {}).items():
+                        if temp_c is None:
+                            continue
+                        cluster_data['nodes'][nm]['temp'] = temp_c
+                        got += 1
+                        if tcache is not None and tlock is not None:
+                            with tlock:
+                                tcache[nm] = {'temp': temp_c, 'ts': now_ts}
+                    # keep the per-manager caches bounded: drop keys for decommissioned/
+                    # renamed nodes so they don't accrete over the process lifetime.
+                    known = set(mgr.nodes or {})
+                    if known:
+                        if tcache is not None and tlock is not None:
+                            with tlock:
+                                for k in [k for k in tcache if k not in known]:
+                                    tcache.pop(k, None)
+                        bo = getattr(mgr, '_node_temp_probe_backoff', None)
+                        if isinstance(bo, dict):
+                            for k in [k for k in bo if k not in known]:
+                                bo.pop(k, None)
+                    if got:
+                        logging.debug(f"[metrics] {cluster_id}: temp for {got}/{len(online_node_names)} node(s)")
+            except Exception as _te:
+                logging.debug(f"[metrics] {cluster_id}: temp collection skipped: {_te}")
+
             # Count VMs + per-VM samples for right-sizing (MK May 2026)
             cluster_data['vms'] = {}  # vmid -> {cpu_pct, mem_pct, maxmem, maxcpu, status}
             try:

pegaprox/core/manager.py

@@ -231,6 +231,12 @@ def __init__(self, cluster_id: str, config: PegaProxConfig):
         self._vm_migration_cooldown = {}  # {vmid: timestamp} — prevent ping-pong
         self._node_metrics_history = {}  # {node_name: [{timestamp, cpu, mem_pct, ...}]} ring buffer for predictive LB
         self._metrics_history_lock = threading.Lock()  # guard concurrent access from API routes
+        # #601 — hottest lm-sensors reading per node (°C), refreshed by the 5-min
+        # metrics collector and READ by the 60s alert loop (which must never SSH
+        # per-tick). _node_temp_probe_backoff skips nodes without lm-sensors/SSH.
+        self._node_temp_cache = {}   # {node_name: {'temp': float_c, 'ts': epoch}}
+        self._node_temp_lock = threading.Lock()
+        self._node_temp_probe_backoff = {}  # {node_name: reprobe_after_epoch}
 
         # maintenance mode
         self.nodes_in_maintenance = {}
@@ -13761,6 +13767,19 @@ def get_node_sensors(self, node: str) -> Dict[str, Any]:
                 out.append(row)
         return {'sensors': out, 'count': len(out)}
 
+    def get_cached_node_temp(self, node: str, max_age: float = 900):
+        """#601 — last cached hottest-sensor temperature (°C) for a node, or None if
+        missing/stale. Populated by the 5-min metrics collector (see background/
+        metrics.py); read by the alert loop so temperature alerts never SSH per-tick."""
+        import time as _t
+        with self._node_temp_lock:
+            ent = self._node_temp_cache.get(node)
+        if not ent:
+            return None
+        if max_age and (_t.time() - ent.get('ts', 0)) > max_age:
+            return None
+        return ent.get('temp')
+
     def get_node_cluster_health(self, node: str) -> Dict[str, Any]:
         # MK May 2026 — SSH-side cluster health: corosync ring status,
         # pvecm quorum + per-service uptime. PVE's REST `/cluster/status`