Ops watchdog: kill activating-window flap + fix two latent bugs

First real Telegram alert from §12.47 was a false positive — the
watchdog tick landed inside the ~1ms window where the
private-watch-poller oneshot was transitioning
inactive/dead → activating/start → active/running → inactive/dead.

Three bugs surfaced + fixed:

1. classifyUnit didn't handle transient legitimate states
   (activating, deactivating, reloading, refreshing, maintenance).
   Now trusts `service.Result=success` from the previous fire
   over the current `ActiveState` for timer-paired oneshots.

2. buildReport had no `unknown` bucket in the degraded
   summary, so the alert read literally `"DEGRADED · "` with
   nothing after the dot. Added the bucket plus a fallback.

3. Staleness check was silently disabled in production.
   `LastTriggerUSec` is human-readable by default, not raw
   microseconds. `Number()` returned NaN so the comparison
   was always false. New `parseSystemdTimestampMs()` handles
   all three known formats.

12 new tests, full suite 535/535 green.

Author: S. K. Rotwang MMMMMMMMM

scripts/seneschal-ops-monitor.mjs

@@ -91,11 +91,21 @@ function systemctlShow(unit) {
 	// doesn't paginate. We also tolerate exit != 0 (e.g. unit not
 	// found) by returning an empty parse, so the classifier sees
 	// status=unknown and the operator gets a clear delta.
+	//
+	// `--timestamp=unix` makes LastTriggerUSec come back as
+	// "@<seconds>.<micros>" instead of "Sun 2026-05-24 09:51 CEST",
+	// which `parseSystemdTimestampMs` handles natively (no Date.parse
+	// timezone-abbreviation guessing required). Older systemd that
+	// doesn't recognise the flag silently ignores it and falls back
+	// to the default human-readable form, which our parser also
+	// handles — so this is a strict improvement either way.
 	try {
 		const out = execFileSync('/usr/bin/systemctl', ['show', unit,
+			'--timestamp=unix',
 			'-p', 'ActiveState',
 			'-p', 'SubState',
 			'-p', 'Result',
+			'-p', 'ExecMainStatus',
 			'-p', 'LastTriggerUSec',
 			'-p', 'NRestarts',
 			'-p', 'UnitFileState'

src/ops-monitor.js

@@ -43,6 +43,57 @@ export function parseShowOutput(text) {
 }
 
 const TIMER_STALE_GRACE_MS = 5 * 60 * 1000;
+// systemd Result values that mean the most recent invocation
+// failed. `success` and `oneshot` (oneshot finished cleanly with
+// RemainAfterExit=no) both mean "ok". Anything else is a failure.
+const FAIL_RESULTS = new Set(['failed', 'exit-code', 'core-dump', 'timeout', 'signal', 'watchdog', 'oom-kill', 'protocol', 'resources']);
+// ActiveStates that are TRANSIENT and legitimate — a tick that
+// happens to land inside one of these (e.g. systemd just told the
+// unit to start, node hasn't booted yet) must not flap the
+// watchdog. We trust `Result` from the previous invocation
+// instead.
+const TRANSIENT_ACTIVE_STATES = new Set(['activating', 'deactivating', 'reloading', 'refreshing', 'maintenance']);
+
+/**
+ * Parse a `LastTriggerUSec` (or any `*USec`) value from
+ * `systemctl show`. Tolerates the three known output formats:
+ *
+ *   * `"1716543662000000"`         — raw microseconds since epoch
+ *                                    (rare, requires `--timestamp=us`
+ *                                    or programmatic D-Bus reads)
+ *   * `"@1716543662.531"`          — `--timestamp=unix` output
+ *   * `"Sun 2026-05-24 09:51:02 CEST"` — default human-readable
+ *
+ * Returns milliseconds since epoch, or `null` if the value is
+ * absent / unparseable. Used by `classifyUnit` for the
+ * timer-staleness check; the prior version silently returned 0
+ * for the human-readable case, defeating the check entirely.
+ */
+export function parseSystemdTimestampMs(value) {
+	if (value == null) return null;
+	const s = String(value).trim();
+	if (!s) return null;
+	// Numeric microseconds since epoch.
+	if (/^\d+$/.test(s)) {
+		const n = Number(s);
+		if (!Number.isFinite(n) || n <= 0) return null;
+		return Math.floor(n / 1000);
+	}
+	// `--timestamp=unix` format: "@<seconds>[.<micros>]".
+	if (s.startsWith('@')) {
+		const seconds = parseFloat(s.slice(1));
+		if (!Number.isFinite(seconds) || seconds <= 0) return null;
+		return Math.round(seconds * 1000);
+	}
+	// Default human-readable. Date.parse handles most timezone
+	// abbreviations on the common platforms; `n/a` and `0` and
+	// other systemd "no value yet" placeholders fall through to
+	// null below.
+	if (s === 'n/a' || s === '0' || s === '-') return null;
+	const ms = Date.parse(s);
+	if (!Number.isFinite(ms) || ms <= 0) return null;
+	return ms;
+}
 
 /**
  * Classify a single unit's parsed `systemctl show` output. The
@@ -55,34 +106,79 @@ const TIMER_STALE_GRACE_MS = 5 * 60 * 1000;
  * `expected.intervalMs` is the expected timer cadence (e.g. 600_000
  * for a 10-minute timer). We add a 5-minute grace before flagging
  * stale.
+ *
+ * Decision tree:
+ *   1. Hard failure signal: `service.Result ∈ FAIL_RESULTS` AND
+ *      `ExecMainStatus !== '0'` → failed. (A oneshot can momentarily
+ *      report `Result=exit-code` even with status=0 on some systemd
+ *      versions; honour the actual exit code.)
+ *   2. Active+failed combo (long-running crash loop): `ActiveState=failed`.
+ *   3. Timer-paired oneshot: trust `Result=success` as definitive
+ *      ok even if the unit is mid-`activating`. This avoids flapping
+ *      every time the watchdog tick races a timer fire.
+ *   4. Timer-paired with stale `LastTriggerUSec` → stale.
+ *   5. Long-running: `ActiveState=active && SubState=running` → ok.
+ *   6. Transient ActiveStates (activating/deactivating/etc) → ok
+ *      with a noted reason; they're never "unknown".
+ *   7. Otherwise unknown (which we still treat as degraded so it
+ *      gets investigated, but doesn't read as a flat-out failure).
  */
 export function classifyUnit({ service, timer, expected, nowMs }) {
 	if (!service || typeof service !== 'object') {
 		return { status: 'unknown', reason: 'no service show data' };
 	}
-	if (service.ActiveState === 'failed' || service.Result === 'failed' || service.Result === 'exit-code') {
-		// Allow Result=exit-code if the service is a oneshot that
-		// finished cleanly (status=0). Distinguish via SubState.
-		if (service.SubState !== 'dead' || service.Result === 'failed' || service.Result === 'exit-code') {
-			if (service.Result !== 'success') {
-				return { status: 'failed', reason: `ActiveState=${service.ActiveState} Result=${service.Result} SubState=${service.SubState}` };
-			}
-		}
+	const exitOk = service.ExecMainStatus == null || String(service.ExecMainStatus) === '0';
+	const resultBad = FAIL_RESULTS.has(service.Result);
+	// 1) + 2): hard failure.
+	if ((resultBad && !exitOk) || service.ActiveState === 'failed') {
+		return {
+			status: 'failed',
+			reason: `ActiveState=${service.ActiveState} Result=${service.Result} ExecMainStatus=${service.ExecMainStatus ?? 'n/a'} SubState=${service.SubState}`
+		};
 	}
+	// 3) + 4): timer-paired logic.
 	if (timer && expected?.intervalMs) {
-		const lastUsec = Number(timer.LastTriggerUSec ?? 0);
-		if (lastUsec > 0) {
-			const lastMs = Math.floor(lastUsec / 1000);
+		const lastMs = parseSystemdTimestampMs(timer.LastTriggerUSec);
+		if (lastMs != null) {
 			const age = nowMs - lastMs;
 			if (age > expected.intervalMs + TIMER_STALE_GRACE_MS) {
-				return { status: 'stale', reason: `timer last fired ${Math.round(age / 60000)} min ago (cadence ${Math.round(expected.intervalMs / 60000)} min)` };
+				return {
+					status: 'stale',
+					reason: `timer last fired ${Math.round(age / 60000)} min ago (cadence ${Math.round(expected.intervalMs / 60000)} min)`
+				};
 			}
 		}
+		// For a timer-paired oneshot, the SERVICE almost always
+		// looks `inactive/dead` between fires. `Result=success`
+		// from the previous fire is the strongest signal we have.
+		if (service.Result === 'success') {
+			return {
+				status: 'ok',
+				reason: `oneshot Result=success ActiveState=${service.ActiveState} SubState=${service.SubState}`
+			};
+		}
 	}
-	if (service.ActiveState === 'active' || service.ActiveState === 'inactive' || service.SubState === 'dead') {
-		return { status: 'ok', reason: `ActiveState=${service.ActiveState} SubState=${service.SubState}` };
+	// 5): long-running pattern.
+	if (service.ActiveState === 'active' && service.SubState === 'running') {
+		return { status: 'ok', reason: 'ActiveState=active SubState=running' };
 	}
-	return { status: 'unknown', reason: `unrecognised state ActiveState=${service.ActiveState} SubState=${service.SubState}` };
+	// Oneshot inactive/dead with success result (e.g. data-rest
+	// after a stop — shouldn't happen but be tolerant).
+	if (service.ActiveState === 'inactive' && service.SubState === 'dead' && service.Result === 'success') {
+		return { status: 'ok', reason: `ActiveState=inactive Result=success` };
+	}
+	// 6): transient legitimate states.
+	if (TRANSIENT_ACTIVE_STATES.has(service.ActiveState)) {
+		return {
+			status: 'ok',
+			reason: `transient ActiveState=${service.ActiveState} SubState=${service.SubState}`
+		};
+	}
+	// 7): genuinely unrecognised.
+	return {
+		status: 'unknown',
+		reason: `unrecognised state ActiveState=${service.ActiveState} Result=${service.Result} SubState=${service.SubState}`
+	};
 }
 
 /**
@@ -99,6 +195,7 @@ export function buildReport({ units = {}, scripts = {}, nowMs }) {
 
 	const failingUnits = unitEntries.filter(([, v]) => v.status === 'failed').map(([k]) => k);
 	const staleUnits = unitEntries.filter(([, v]) => v.status === 'stale').map(([k]) => k);
+	const unknownUnits = unitEntries.filter(([, v]) => v.status === 'unknown').map(([k]) => k);
 	const missingScripts = scriptEntries.filter(([, v]) => v.status !== 'ok').map(([k]) => k);
 
 	let summary;
@@ -109,7 +206,15 @@ export function buildReport({ units = {}, scripts = {}, nowMs }) {
 		const parts = [];
 		if (failingUnits.length) parts.push(`${failingUnits.length} failed (${failingUnits.join(', ')})`);
 		if (staleUnits.length) parts.push(`${staleUnits.length} stale (${staleUnits.join(', ')})`);
+		if (unknownUnits.length) parts.push(`${unknownUnits.length} unknown (${unknownUnits.join(', ')})`);
 		if (missingScripts.length) parts.push(`${missingScripts.length} script(s) missing (${missingScripts.join(', ')})`);
+		// Defensive: if we somehow flipped to degraded with no
+		// recognised buckets, surface that explicitly rather than
+		// printing "DEGRADED · " (the bug the watchdog itself caught
+		// on its first alert!).
+		if (parts.length === 0) {
+			parts.push(`${unitEntries.length} units + ${scriptEntries.length} scripts checked, no recognised failure bucket — investigate the full report`);
+		}
 		summary = `DEGRADED · ${parts.join('; ')}`;
 	}
 	return Object.freeze({