Ops watchdog: catch silent service failures within 15 minutes

Background: during the previous deploy sweep we caught three
seneschal-* systemd units that had been failing every tick for
~36 hours because /opt/seneschal-data-api/scripts/*.mjs went
missing (rsync regression). Nothing was watching systemd state,
so the income-poller lost ~36 hourly snapshots before anyone
noticed.

* seneschal-ops-monitor.{mjs,service,timer}: 10-minute watchdog
  that parses `systemctl show` for every seneschal-* unit +
  paired timer, stats the expected scripts under /opt/.../scripts,
  classifies state (ok/failed/stale/missing/unknown), and posts
  state-change deltas to Telegram (silent no-op when creds
  unconfigured).
* ops-monitor.js: pure helpers (parser, classifier, report
  builder, diff, Telegram POST) — fully unit-tested without
  systemd or network round-trips.
* /v1/ops/health endpoint exposes the watchdog state file. 200
  when ok, 503 when degraded/stale/unknown so external monitors
  can alert on the meta-failure of the watchdog itself.
* stats.seneschal.space: new "Ops watchdog" tile with overall
  state + coloured-dot strip showing every unit + script
  tracked. Hover tooltips spell out the systemd reason.
* 32 new tests (28 ops-monitor + 4 /v1/ops/health), full suite
  523/523 green.

Author: S. K. Rotwang MMMMMMMMM

docs/stats.html

@@ -280,6 +280,22 @@ <h2 style="color:#9d6cff;margin:0">Private Watch &mdash; view-key payment monito
 		<h2 style="color:var(--accent);margin:0">Seneschal activity — last 24h</h2>
 		<span class="kpi-sub">counts only — no profit figures exposed</span>
 	</header>
+
+	<!-- Ops health strip — populated by /v1/ops/health on every
+	     stats refresh. Each unit gets a coloured dot so a regression
+	     is visible at a glance from the public dashboard. -->
+	<section id="ops-health" class="kpi-grid" style="margin-bottom:1.4em">
+		<div class="op-tile" style="grid-column:span 2">
+			<div class="op-label">Ops watchdog</div>
+			<div class="op-value mono-sm" id="ops-overall">—</div>
+			<div class="op-sub" id="ops-summary">checked every 10 min · post-mortem alerts via @OrknetP</div>
+		</div>
+		<div class="op-tile" style="grid-column:span 4">
+			<div class="op-label">Per-unit status</div>
+			<div class="op-value" id="ops-dots" style="display:flex;flex-wrap:wrap;gap:0.45em;font-size:0.85em;line-height:1.4;align-items:center">—</div>
+			<div class="op-sub" id="ops-changes">last change: none recorded since boot</div>
+		</div>
+	</section>
 	<div class="operator-tiles">
 		<div class="op-tile">
 			<div class="op-label">Liquidation attempts</div>
@@ -1119,6 +1135,53 @@ <h2>Support development</h2>
 	}[c]));
 }
 
+// Render the ops-health strip. Splits the watchdog report into a
+// summary tile + a per-unit dot row. We tolerate any failure mode
+// of the endpoint (404, 503, network) by colouring the dot row to
+// match — `degraded`/`stale`/`unknown` all show as their own dot.
+async function renderOpsHealth() {
+	const overallEl = document.getElementById('ops-overall');
+	const summaryEl = document.getElementById('ops-summary');
+	const dotsEl = document.getElementById('ops-dots');
+	const changesEl = document.getElementById('ops-changes');
+	if (!overallEl) return;
+	let report = null;
+	let httpStatus = 0;
+	try {
+		const r = await fetch(`${API_BASE}/v1/ops/health`, { headers: { accept: 'application/json' } });
+		httpStatus = r.status;
+		report = await r.json();
+	}
+	catch (e) {
+		overallEl.textContent = 'UNREACHABLE';
+		overallEl.className = 'op-value mono-sm warn';
+		summaryEl.textContent = `couldn't reach /v1/ops/health: ${e.message}`;
+		dotsEl.textContent = '—';
+		return;
+	}
+	const overall = report?.overall ?? 'unknown';
+	const colour = { ok: 'good', degraded: 'warn', stale: 'warn', unknown: 'warn' }[overall] ?? 'warn';
+	overallEl.className = 'op-value mono-sm ' + colour;
+	overallEl.textContent = overall.toUpperCase();
+	const ageMin = report?.ageMs ? Math.round(report.ageMs / 60000) : null;
+	summaryEl.textContent = `${report?.summary ?? '(no summary)'}${ageMin != null ? ` · last tick ${ageMin}m ago` : ''}`;
+	const units = report?.units ?? {};
+	const scripts = report?.scripts ?? {};
+	const dotFor = (status) => {
+		const c = { ok: 'var(--good)', failed: 'var(--danger)', stale: 'var(--warn)', missing: 'var(--danger)', unknown: 'var(--muted)' }[status] ?? 'var(--muted)';
+		return `<span style="display:inline-block;width:0.65em;height:0.65em;border-radius:50%;background:${c}"></span>`;
+	};
+	const shortName = (n) => n.replace(/^seneschal-/, '').replace(/\.service$/, '');
+	const unitBadges = Object.entries(units).map(([name, v]) =>
+		`<span style="display:inline-flex;gap:0.35em;align-items:center" title="${escapeHtml(v.reason ?? v.status)}">${dotFor(v.status)} ${escapeHtml(shortName(name))}</span>`
+	);
+	const scriptBadges = Object.entries(scripts).map(([path, v]) => {
+		const base = path.split('/').pop();
+		return `<span style="display:inline-flex;gap:0.35em;align-items:center" title="${escapeHtml(v.reason ?? v.status)}">${dotFor(v.status)} ${escapeHtml(base)}</span>`;
+	});
+	dotsEl.innerHTML = unitBadges.concat(scriptBadges).join(' · ');
+}
+
 // ── data fetch + render orchestration ─────────────────────
 async function refresh() {
 	setStatus('loading', 'refreshing…');
@@ -1138,12 +1201,18 @@ <h2>Support development</h2>
 		renderRecentLiq(d);
 		renderPremium(d);
 		renderSupport(d);
+		// Ops-health is its own endpoint so a slow stats overview
+		// doesn't gate the operational view (or vice-versa).
+		renderOpsHealth().catch(() => { /* already painted error state */ });
 		clearError();
 		const ts = new Date(d.as_of_ms);
 		setStatus('', `updated ${ts.toLocaleTimeString()}`);
 	} catch (e) {
 		setStatus('error', `error: ${e.message}`);
 		showError(`Could not load stats: ${e.message}. Retrying in 30s.`);
+		// Still paint ops-health independently — if the stats DB
+		// is down we still want to know whether the watchdog is up.
+		renderOpsHealth().catch(() => {});
 	}
 }
 

scripts/seneschal-ops-monitor.env.example

@@ -0,0 +1,37 @@
+# Seneschal ops watchdog config — operator-managed env.
+#
+# Install location on fin4: /etc/seneschal/ops-monitor.env
+# Mode: 600 root:root
+#
+# Everything below is optional. With nothing set the watchdog still
+# runs (every 10 min via seneschal-ops-monitor.timer), writes its
+# state to /var/lib/seneschal-ops-monitor/state.json (consumed by
+# /v1/ops/health and the stats page) and logs to journalctl. The
+# only thing you lose is push notifications.
+
+# ── Telegram alerting (recommended) ─────────────────────────────
+# 1. Talk to @BotFather, /newbot, follow prompts. Save the token.
+# 2. Start a chat with your new bot from @OrknetP (the bot has to
+#    have spoken to you first before it can DM you). Send any
+#    message.
+# 3. curl "https://api.telegram.org/bot$BOT_TOKEN/getUpdates" — find
+#    your chat in the `result[*].message.chat.id` field. Numeric
+#    integer for private chats, negative for group chats.
+# 4. Drop them in here, chmod 600, then `systemctl start
+#    seneschal-ops-monitor.service` to force a fresh tick — you'll
+#    get an "Initial watchdog state" message if anything's degraded,
+#    nothing otherwise.
+#
+# TELEGRAM_BOT_TOKEN=123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11
+# TELEGRAM_CHAT_ID=123456789
+
+# ── Optional: override the units / scripts that are checked ─────
+# Comma-separated. For unit names append ":cadenceSec" to enable
+# the timer-staleness check (omit the colon for long-running
+# services like the REST server).
+#
+# SENESCHAL_OPS_UNITS=seneschal-data-rest.service,seneschal-private-watch-poller.service:180,seneschal-income-poller.service:3600
+# SENESCHAL_OPS_SCRIPTS=/opt/seneschal-data-api/scripts/private-watch-poller.mjs,/opt/seneschal-data-api/scripts/income-poller.mjs
+
+# ── Optional: state file path ───────────────────────────────────
+# SENESCHAL_OPS_STATE_FILE=/var/lib/seneschal-ops-monitor/state.json

scripts/seneschal-ops-monitor.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+// Seneschal ops watchdog.
+//
+// Inspects the seneschal-* systemd units + the data-api script
+// directory, builds a health report, persists it, and alerts on
+// state changes via:
+//   * Always: writes the JSON report to STATE_FILE
+//             (default /var/lib/seneschal-ops-monitor/state.json),
+//             and logs a one-line summary to stdout (captured by
+//             systemd-journald).
+//   * Optional: posts state-change deltas to Telegram if both
+//     TELEGRAM_BOT_TOKEN + TELEGRAM_CHAT_ID env vars are set.
+//
+// Reads its config from env so the same binary runs in dev + prod:
+//   SENESCHAL_OPS_UNITS       — comma list of "service[:cadenceSec]"
+//                               pairs. Cadence is the expected
+//                               timer interval; omit (no colon) for
+//                               long-running services.
+//   SENESCHAL_OPS_SCRIPTS     — comma list of absolute script
+//                               paths to verify exist + are
+//                               executable. Default is the four
+//                               .mjs/.sh scripts the data-api
+//                               relies on.
+//   SENESCHAL_OPS_STATE_FILE  — where to persist the last report
+//                               (default /var/lib/seneschal-ops-monitor/state.json)
+//   TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID — optional alert sink.
+//
+// Exit codes:
+//   0 — overall ok
+//   2 — degraded (one or more units / scripts unhealthy)
+//   1 — watchdog itself crashed (uncaught error)
+
+import { execFileSync } from 'node:child_process';
+import { readFile, writeFile, mkdir, stat, access } from 'node:fs/promises';
+import { constants as FS } from 'node:fs';
+import { dirname } from 'node:path';
+import { hostname } from 'node:os';
+
+import {
+	parseShowOutput,
+	classifyUnit,
+	buildReport,
+	diffReports,
+	renderTelegramMessage,
+	sendTelegram
+} from '../src/ops-monitor.js';
+
+const DEFAULTS = Object.freeze({
+	UNITS: [
+		// timer-driven oneshots (cadenceSec → interval grace)
+		{ name: 'seneschal-private-watch-poller.service', timerName: 'seneschal-private-watch-poller.timer', intervalSec: 180 },
+		{ name: 'seneschal-income-poller.service',         timerName: 'seneschal-income-poller.timer',         intervalSec: 3600 },
+		{ name: 'seneschal-paymaster-sweep-check.service', timerName: 'seneschal-paymaster-sweep-check.timer', intervalSec: 86_400 },
+		{ name: 'seneschal-backup.service',                timerName: 'seneschal-backup.timer',                intervalSec: 86_400 },
+		// long-running services (no timer pair)
+		{ name: 'seneschal-data-rest.service' },
+		{ name: 'seneschal-data-mcp.service' }
+	],
+	SCRIPTS: [
+		'/opt/seneschal-data-api/scripts/private-watch-poller.mjs',
+		'/opt/seneschal-data-api/scripts/income-poller.mjs',
+		'/opt/seneschal-data-api/scripts/paymaster-sweep-check.mjs',
+		'/opt/seneschal-data-api/scripts/publish-docs.sh'
+	],
+	STATE_FILE: '/var/lib/seneschal-ops-monitor/state.json'
+});
+
+function parseUnitList(envValue) {
+	if (!envValue) return DEFAULTS.UNITS;
+	return envValue.split(',').map(s => s.trim()).filter(Boolean).map(s => {
+		const [name, intervalSec] = s.split(':');
+		// Convention: if there's a paired timer with the same stem,
+		// the operator should add `:cadenceSec`; otherwise we treat
+		// it as a long-running service.
+		const out = { name };
+		if (intervalSec) {
+			out.timerName = name.replace(/\.service$/, '.timer');
+			out.intervalSec = parseInt(intervalSec, 10);
+		}
+		return out;
+	});
+}
+
+function parseScriptList(envValue) {
+	if (!envValue) return DEFAULTS.SCRIPTS;
+	return envValue.split(',').map(s => s.trim()).filter(Boolean);
+}
+
+function systemctlShow(unit) {
+	// We deliberately do NOT use --no-pager — `systemctl show`
+	// doesn't paginate. We also tolerate exit != 0 (e.g. unit not
+	// found) by returning an empty parse, so the classifier sees
+	// status=unknown and the operator gets a clear delta.
+	try {
+		const out = execFileSync('/usr/bin/systemctl', ['show', unit,
+			'-p', 'ActiveState',
+			'-p', 'SubState',
+			'-p', 'Result',
+			'-p', 'LastTriggerUSec',
+			'-p', 'NRestarts',
+			'-p', 'UnitFileState'
+		], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+		return parseShowOutput(out);
+	}
+	catch {
+		return {};
+	}
+}
+
+async function checkScript(path) {
+	try {
+		const s = await stat(path);
+		if (!s.isFile()) return { status: 'missing', reason: 'not a regular file' };
+		try {
+			await access(path, FS.X_OK);
+		}
+		catch {
+			return { status: 'missing', reason: 'not executable' };
+		}
+		return { status: 'ok', sizeBytes: s.size, mtimeMs: s.mtimeMs };
+	}
+	catch (err) {
+		return { status: 'missing', reason: err?.code ?? String(err) };
+	}
+}
+
+async function loadPriorReport(path) {
+	try {
+		const buf = await readFile(path, 'utf8');
+		return JSON.parse(buf);
+	}
+	catch {
+		return null;
+	}
+}
+
+async function savePriorReport(path, report) {
+	await mkdir(dirname(path), { recursive: true });
+	await writeFile(path, JSON.stringify(report, null, 2));
+}
+
+async function main() {
+	const nowMs = Date.now();
+	const units = parseUnitList(process.env.SENESCHAL_OPS_UNITS);
+	const scripts = parseScriptList(process.env.SENESCHAL_OPS_SCRIPTS);
+	const stateFile = process.env.SENESCHAL_OPS_STATE_FILE ?? DEFAULTS.STATE_FILE;
+
+	const unitClassifications = {};
+	for (const u of units) {
+		const serviceShow = systemctlShow(u.name);
+		const timerShow = u.timerName ? systemctlShow(u.timerName) : null;
+		unitClassifications[u.name] = classifyUnit({
+			service: serviceShow,
+			timer: timerShow,
+			expected: u.intervalSec ? { intervalMs: u.intervalSec * 1000 } : null,
+			nowMs
+		});
+	}
+
+	const scriptChecks = {};
+	for (const p of scripts) {
+		scriptChecks[p] = await checkScript(p);
+	}
+
+	const report = buildReport({
+		units: unitClassifications,
+		scripts: scriptChecks,
+		nowMs
+	});
+
+	const prior = await loadPriorReport(stateFile);
+	const changes = diffReports(prior, report);
+
+	// Always log the summary so journalctl tells a continuous
+	// story even when nothing changed.
+	console.log(JSON.stringify({
+		t: new Date(nowMs).toISOString(),
+		event: 'ops_watchdog_tick',
+		overall: report.overall,
+		summary: report.summary,
+		changes
+	}));
+
+	if (changes.length > 0) {
+		console.warn(JSON.stringify({
+			t: new Date(nowMs).toISOString(),
+			event: 'ops_watchdog_change',
+			level: report.overall === 'ok' ? 'recovery' : 'alert',
+			changes
+		}));
+		const text = renderTelegramMessage({
+			changes,
+			report,
+			host: hostname()
+		});
+		const tg = await sendTelegram({
+			botToken: process.env.TELEGRAM_BOT_TOKEN,
+			chatId: process.env.TELEGRAM_CHAT_ID,
+			text
+		});
+		console.log(JSON.stringify({
+			t: new Date(nowMs).toISOString(),
+			event: 'ops_watchdog_alert',
+			telegram: tg
+		}));
+	}
+
+	await savePriorReport(stateFile, report);
+	process.exit(report.overall === 'ok' ? 0 : 2);
+}
+
+main().catch(err => {
+	console.error(JSON.stringify({
+		t: new Date().toISOString(),
+		event: 'ops_watchdog_crash',
+		error: err?.stack ?? err?.message ?? String(err)
+	}));
+	process.exit(1);
+});