|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +"""In-band BMC / hardware health via local `ipmitool` (#609). |
| 3 | +
|
| 4 | +CREDENTIAL-FREE by design: instead of storing BMC network credentials and |
| 5 | +reaching the BMC over its management network (the "crown-jewels" path — see the |
| 6 | +Redfish opt-in phase), we SSH to the PVE node — access PegaProx already has for |
| 7 | +lm-sensors / fencing — and run `ipmitool` LOCALLY there. The host talks to its |
| 8 | +own BMC over the internal KCS interface (`/dev/ipmi0`) with no username/password |
| 9 | +and without bridging the data network to the out-of-band management plane. |
| 10 | +
|
| 11 | +Read-only: only `sdr` / `sel` / `fru` / `dcmi` / `chassis status` are issued — |
| 12 | +no power / virtual-media / firmware commands. Availability is opportunistic: |
| 13 | +if `ipmitool` or `/dev/ipmi0` is absent we return `available=False` and change |
| 14 | +nothing on the host (PegaProx never auto-enables the IPMI channel — the optional |
| 15 | +install + the compliance acknowledgement are gated at the API layer). |
| 16 | +
|
| 17 | +Parsers are pure (text -> dict) so they can be fixture-tested without hardware. |
| 18 | +""" |
| 19 | + |
| 20 | +import re |
| 21 | + |
| 22 | +# Markers so the whole read is ONE SSH round-trip; the orchestrator splits on them. |
| 23 | +_M_SENS, _M_CHAS, _M_POWER, _M_FRU, _M_SEL = ( |
| 24 | + '__PP_SENSORS__', '__PP_CHASSIS__', '__PP_POWER__', '__PP_FRU__', '__PP_SEL__') |
| 25 | + |
| 26 | +# Single shell script run on the node. Bails early (no host mutation) when the |
| 27 | +# in-band interface is missing, so we never touch a hardened box that removed it. |
| 28 | +INBAND_PROBE_CMD = ( |
| 29 | + "if ! command -v ipmitool >/dev/null 2>&1; then echo __PP_NO_IPMITOOL__; exit 0; fi; " |
| 30 | + "if ! ipmitool mc info >/dev/null 2>&1; then echo __PP_NO_BMC__; exit 0; fi; " |
| 31 | + "echo " + _M_SENS + "; ipmitool sdr elist 2>/dev/null; " |
| 32 | + "echo " + _M_CHAS + "; ipmitool chassis status 2>/dev/null; " |
| 33 | + "echo " + _M_POWER + "; ipmitool dcmi power reading 2>/dev/null; " |
| 34 | + "echo " + _M_FRU + "; ipmitool fru print 0 2>/dev/null; " |
| 35 | + "echo " + _M_SEL + "; ipmitool sel elist last 25 2>/dev/null" |
| 36 | +) |
| 37 | + |
| 38 | +# ipmitool sdr status tokens -> normalized health |
| 39 | +_SDR_STATUS = {'ok': 'ok', 'ns': 'na', 'nc': 'warning', 'cr': 'critical', 'nr': 'critical'} |
| 40 | + |
| 41 | + |
| 42 | +def _num(s): |
| 43 | + """First numeric token in a string as float, or None.""" |
| 44 | + m = re.search(r'-?\d+(?:\.\d+)?', s or '') |
| 45 | + return float(m.group(0)) if m else None |
| 46 | + |
| 47 | + |
| 48 | +def parse_sensors(text): |
| 49 | + """`ipmitool sdr elist` -> [{name, kind, value, unit, reading, status}]. |
| 50 | +
|
| 51 | + Line shape: ``Name | 01h | ok | 3.1 | 45 degrees C`` (pipe-separated, 5 cols). |
| 52 | + kind is derived from the reading unit (temp / fan / volt / power / other). |
| 53 | + """ |
| 54 | + out = [] |
| 55 | + for line in (text or '').splitlines(): |
| 56 | + if '|' not in line: |
| 57 | + continue |
| 58 | + cols = [c.strip() for c in line.split('|')] |
| 59 | + if len(cols) < 5: |
| 60 | + continue |
| 61 | + name, status_tok, reading = cols[0], cols[2].lower(), cols[4] |
| 62 | + if not name: |
| 63 | + continue |
| 64 | + rl = reading.lower() |
| 65 | + if 'degrees c' in rl or rl.endswith(' c') or 'degree' in rl: |
| 66 | + kind, unit = 'temp', '°C' |
| 67 | + elif 'rpm' in rl: |
| 68 | + kind, unit = 'fan', 'RPM' |
| 69 | + elif 'volt' in rl: |
| 70 | + kind, unit = 'volt', 'V' |
| 71 | + elif 'watt' in rl: |
| 72 | + kind, unit = 'power', 'W' |
| 73 | + elif 'amp' in rl: |
| 74 | + kind, unit = 'current', 'A' |
| 75 | + else: |
| 76 | + kind, unit = 'discrete', '' |
| 77 | + out.append({ |
| 78 | + 'name': name, |
| 79 | + 'kind': kind, |
| 80 | + 'value': _num(reading) if kind not in ('discrete',) else None, |
| 81 | + 'unit': unit, |
| 82 | + 'reading': reading, |
| 83 | + 'status': _SDR_STATUS.get(status_tok, status_tok or 'na'), |
| 84 | + }) |
| 85 | + return out |
| 86 | + |
| 87 | + |
| 88 | +def parse_chassis(text): |
| 89 | + """`ipmitool chassis status` -> {power, intrusion, fault, ...}.""" |
| 90 | + d = {} |
| 91 | + for line in (text or '').splitlines(): |
| 92 | + if ':' not in line: |
| 93 | + continue |
| 94 | + k, v = line.split(':', 1) |
| 95 | + k, v = k.strip().lower(), v.strip() |
| 96 | + if k == 'system power': |
| 97 | + d['power'] = v |
| 98 | + elif 'intrusion' in k: |
| 99 | + d['intrusion'] = v |
| 100 | + elif 'fault' in k and v: |
| 101 | + d.setdefault('faults', []).append(f"{k}: {v}") |
| 102 | + return d |
| 103 | + |
| 104 | + |
| 105 | +def parse_power(text): |
| 106 | + """`ipmitool dcmi power reading` -> instantaneous watts (float) or None.""" |
| 107 | + for line in (text or '').splitlines(): |
| 108 | + if 'instantaneous power reading' in line.lower(): |
| 109 | + return _num(line) |
| 110 | + return None |
| 111 | + |
| 112 | + |
| 113 | +def parse_fru(text): |
| 114 | + """`ipmitool fru print 0` -> {manufacturer, product, serial, part, board_*}.""" |
| 115 | + kv = {} |
| 116 | + for line in (text or '').splitlines(): |
| 117 | + if ':' not in line: |
| 118 | + continue |
| 119 | + k, v = line.split(':', 1) |
| 120 | + kv[k.strip().lower()] = v.strip() |
| 121 | + g = lambda *keys: next((kv[k] for k in keys if kv.get(k)), '') |
| 122 | + return { |
| 123 | + 'manufacturer': g('product manufacturer', 'board mfg', 'chassis manufacturer'), |
| 124 | + 'product': g('product name', 'board product'), |
| 125 | + 'serial': g('product serial', 'board serial', 'chassis serial'), |
| 126 | + 'part': g('product part number', 'board part number'), |
| 127 | + } |
| 128 | + |
| 129 | + |
| 130 | +def parse_sel(text, limit=25): |
| 131 | + """`ipmitool sel elist` -> recent hardware events (newest first). |
| 132 | +
|
| 133 | + Line shape: ``12 | 07/14/2026 | 21:30:05 | Power Supply PSU2 | Failure detected | Asserted`` |
| 134 | + """ |
| 135 | + events = [] |
| 136 | + for line in (text or '').splitlines(): |
| 137 | + if '|' not in line: |
| 138 | + continue |
| 139 | + cols = [c.strip() for c in line.split('|')] |
| 140 | + if len(cols) < 4: |
| 141 | + continue |
| 142 | + # cols: id, date, time, sensor, [description], [state] |
| 143 | + ts = (cols[1] + ' ' + cols[2]).strip() if len(cols) >= 3 else '' |
| 144 | + sensor = cols[3] if len(cols) > 3 else '' |
| 145 | + desc = cols[4] if len(cols) > 4 else '' |
| 146 | + state = cols[5] if len(cols) > 5 else '' |
| 147 | + low = (desc + ' ' + state + ' ' + sensor).lower() |
| 148 | + sev = 'critical' if any(w in low for w in ('fail', 'critical', 'fault', 'error', 'lost')) \ |
| 149 | + else 'warning' if any(w in low for w in ('warn', 'non-critical', 'degrad', 'intrusion')) \ |
| 150 | + else 'info' |
| 151 | + events.append({'id': cols[0], 'time': ts, 'sensor': sensor, |
| 152 | + 'description': (desc + (' — ' + state if state else '')).strip(' —'), |
| 153 | + 'severity': sev}) |
| 154 | + events.reverse() # newest first |
| 155 | + return events[:limit] |
| 156 | + |
| 157 | + |
| 158 | +def _health_rollup(sensors, sel, chassis): |
| 159 | + """Overall node hardware health from the parsed pieces: ok / warning / critical.""" |
| 160 | + if any(s['status'] == 'critical' for s in sensors) or \ |
| 161 | + any(e['severity'] == 'critical' for e in sel) or \ |
| 162 | + (chassis.get('intrusion', '').lower() not in ('', 'inactive', 'not present', 'disabled')): |
| 163 | + return 'critical' |
| 164 | + if any(s['status'] == 'warning' for s in sensors) or \ |
| 165 | + any(e['severity'] == 'warning' for e in sel): |
| 166 | + return 'warning' |
| 167 | + return 'ok' |
| 168 | + |
| 169 | + |
| 170 | +def parse_inband(raw): |
| 171 | + """Split the marker-delimited SSH output and parse every section. |
| 172 | +
|
| 173 | + Returns {'available': bool, 'reason'?: str, 'health', 'sensors', 'chassis', |
| 174 | + 'power_w', 'fru', 'events'}. |
| 175 | + """ |
| 176 | + raw = raw or '' |
| 177 | + if '__PP_NO_IPMITOOL__' in raw: |
| 178 | + return {'available': False, 'reason': 'ipmitool is not installed on this node'} |
| 179 | + if '__PP_NO_BMC__' in raw: |
| 180 | + return {'available': False, 'reason': 'no in-band BMC / IPMI interface (/dev/ipmi0) on this node'} |
| 181 | + |
| 182 | + def section(marker, nxt): |
| 183 | + try: |
| 184 | + body = raw.split(marker, 1)[1] |
| 185 | + except IndexError: |
| 186 | + return '' |
| 187 | + for m in nxt: |
| 188 | + if m in body: |
| 189 | + body = body.split(m, 1)[0] |
| 190 | + return body |
| 191 | + |
| 192 | + sensors = parse_sensors(section(_M_SENS, [_M_CHAS, _M_POWER, _M_FRU, _M_SEL])) |
| 193 | + chassis = parse_chassis(section(_M_CHAS, [_M_POWER, _M_FRU, _M_SEL])) |
| 194 | + power_w = parse_power(section(_M_POWER, [_M_FRU, _M_SEL])) |
| 195 | + fru = parse_fru(section(_M_FRU, [_M_SEL])) |
| 196 | + events = parse_sel(section(_M_SEL, [])) |
| 197 | + if not (sensors or chassis or events or power_w is not None or any(fru.values())): |
| 198 | + return {'available': False, 'reason': 'in-band BMC returned no data'} |
| 199 | + return { |
| 200 | + 'available': True, |
| 201 | + 'health': _health_rollup(sensors, sel=events, chassis=chassis), |
| 202 | + 'sensors': sensors, |
| 203 | + 'chassis': chassis, |
| 204 | + 'power_w': power_w, |
| 205 | + 'fru': fru, |
| 206 | + 'events': events, |
| 207 | + } |
| 208 | + |
| 209 | + |
| 210 | +def read_node_bmc_inband(mgr, node, timeout=15): |
| 211 | + """Orchestrator: SSH to `node`, run the read-only ipmitool probe, parse it. |
| 212 | +
|
| 213 | + Credential-free (in-band). Returns the parse_inband() dict, or an |
| 214 | + {'available': False, 'reason': ...} on any SSH/resolution failure. Never |
| 215 | + mutates the node. The compliance acknowledgement + optional install are |
| 216 | + enforced by the caller (API layer), not here. |
| 217 | + """ |
| 218 | + try: |
| 219 | + ip = mgr._get_node_ip(node) |
| 220 | + if not ip: |
| 221 | + return {'available': False, 'reason': f'no SSH-reachable IP for node {node}'} |
| 222 | + user = getattr(mgr.config, 'ssh_user', None) or 'root' |
| 223 | + raw = mgr._ssh_run_command_output(ip, user, INBAND_PROBE_CMD, timeout=timeout) |
| 224 | + if raw is None or not raw.strip(): |
| 225 | + return {'available': False, 'reason': 'no response from node (SSH unavailable?)'} |
| 226 | + return parse_inband(raw) |
| 227 | + except Exception as e: # noqa: BLE001 — surface as unavailable, never raise into the route |
| 228 | + return {'available': False, 'reason': f'in-band BMC read failed: {e}'} |
0 commit comments