diff --git a/.aios-core/core/health-check/proactive-sentinel.js b/.aios-core/core/health-check/proactive-sentinel.js new file mode 100644 index 0000000000..9faf969d38 --- /dev/null +++ b/.aios-core/core/health-check/proactive-sentinel.js @@ -0,0 +1,731 @@ +#!/usr/bin/env node + +/** + * AIOX Proactive Sentinel + * + * Story: HCS-3 - Proactive Health Monitoring + * Epic: Epic 9 - Persistent Memory Layer / Health Check System + * + * Continuously monitors system health indicators and triggers + * preventive actions before failures occur. Works with HealerManager + * for automated remediation. + * + * Features: + * - AC1: Registers watchpoints for system health indicators + * - AC2: Evaluates watchpoints on configurable intervals + * - AC3: Fires alerts with severity levels (info, warning, critical) + * - AC4: Tracks alert history for pattern detection + * - AC5: Integrates with HealerManager for auto-remediation + * - AC6: Supports custom watchpoint functions + * - AC7: Provides health score (0-100) aggregated from all watchpoints + * - AC8: Persists alert history in .aiox/sentinel-alerts.json + * + * @author @dev (Dex) + * @version 1.0.0 + */ + +const fs = require('fs'); +const path = require('path'); +const EventEmitter = require('events'); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// CONFIGURATION +// ═══════════════════════════════════════════════════════════════════════════════════ + +const CONFIG = { + alertsPath: '.aiox/sentinel-alerts.json', + maxAlerts: 1000, + retentionDays: 30, + defaultIntervalMs: 60000, // 1 minute + healthScoreWeights: { + critical: 30, + warning: 10, + info: 2, + }, + version: '1.0.0', + schemaVersion: 'aiox-sentinel-v1', +}; + +// ═══════════════════════════════════════════════════════════════════════════════════ +// ENUMS +// ═══════════════════════════════════════════════════════════════════════════════════ + +const AlertSeverity = { + INFO: 'info', + WARNING: 'warning', + CRITICAL: 'critical', +}; + +const WatchpointStatus = { + HEALTHY: 'healthy', + DEGRADED: 'degraded', + FAILING: 'failing', + UNKNOWN: 'unknown', +}; + +const Events = { + ALERT_FIRED: 'sentinel:alert', + WATCHPOINT_REGISTERED: 'sentinel:watchpoint:registered', + WATCHPOINT_EVALUATED: 'sentinel:watchpoint:evaluated', + HEALTH_SCORE_CHANGED: 'sentinel:health:changed', + SENTINEL_STARTED: 'sentinel:started', + SENTINEL_STOPPED: 'sentinel:stopped', + ALERTS_PRUNED: 'sentinel:alerts:pruned', +}; + +// ═══════════════════════════════════════════════════════════════════════════════════ +// PROACTIVE SENTINEL +// ═══════════════════════════════════════════════════════════════════════════════════ + +class ProactiveSentinel extends EventEmitter { + /** + * @param {Object} options + * @param {string} [options.projectRoot] - Project root directory + * @param {Object} [options.config] - Override default config + */ + constructor(options = {}) { + super(); + this.projectRoot = options.projectRoot ?? process.cwd(); + this.config = { ...CONFIG, ...options.config }; + this.watchpoints = new Map(); + this.alerts = []; + this.running = false; + this._timers = new Map(); + this._lastHealthScore = 100; + this._loaded = false; + this._saving = false; + } + + /** + * Get the alerts file path + * @returns {string} Absolute path to sentinel-alerts.json + * @private + */ + _getFilePath() { + return path.join(this.projectRoot, this.config.alertsPath); + } + + /** + * Ensure persisted state has been hydrated before operating. + * Lazy-loads on first access to prevent empty-buffer overwrites. + * @returns {Promise} + * @private + */ + async _ensureLoaded() { + if (!this._loaded) { + await this.load(); + } + } + + /** + * Load alert history from disk. + * Logs a warning on schema mismatch or parse errors. + * @returns {Promise} + */ + async load() { + const filePath = this._getFilePath(); + try { + if (fs.existsSync(filePath)) { + const raw = fs.readFileSync(filePath, 'utf8'); + const data = JSON.parse(raw); + + if (data.schemaVersion !== this.config.schemaVersion) { + console.warn( + `[ProactiveSentinel] Schema mismatch: found '${data.schemaVersion}', expected '${this.config.schemaVersion}'. Resetting alerts.`, + ); + this.alerts = []; + this._loaded = true; + return; + } + + this.alerts = Array.isArray(data.alerts) ? data.alerts : []; + } + } catch (err) { + console.warn('[ProactiveSentinel] Failed to load alerts:', err.message); + this.alerts = []; + } + this._loaded = true; + } + + /** + * Save alert history to disk. + * Guards against concurrent writes and uses atomic write (tmp + rename). + * @returns {Promise} + */ + async save() { + if (this._saving) return; + this._saving = true; + + const filePath = this._getFilePath(); + const dir = path.dirname(filePath); + + try { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const data = { + schemaVersion: this.config.schemaVersion, + version: this.config.version, + savedAt: new Date().toISOString(), + alerts: this.alerts, + }; + + const tmpPath = `${filePath}.tmp`; + fs.writeFileSync(tmpPath, JSON.stringify(data, null, 2), 'utf8'); + fs.renameSync(tmpPath, filePath); + } catch (err) { + console.error('[ProactiveSentinel] Failed to save alerts:', err.message); + } finally { + this._saving = false; + } + } + + /** + * Register a watchpoint + * + * @param {Object} watchpoint + * @param {string} watchpoint.id - Unique watchpoint ID + * @param {string} watchpoint.name - Human-readable name + * @param {string} watchpoint.description - What this monitors + * @param {Function} watchpoint.check - Async function returning { status, message, data } + * @param {number} [watchpoint.intervalMs] - Check interval (default: 60000) + * @param {string} [watchpoint.severity] - Alert severity when failing (default: warning) + * @param {boolean} [watchpoint.autoHeal] - Whether to trigger auto-healing (default: false) + * @param {string} [watchpoint.healerId] - HealerManager check ID for remediation + * @returns {Object} The registered watchpoint entry + */ + registerWatchpoint(watchpoint) { + if (!watchpoint.id || !watchpoint.name || !watchpoint.check) { + throw new Error('Required fields: id, name, check'); + } + + if (typeof watchpoint.check !== 'function') { + throw new Error('check must be a function'); + } + + const entry = { + id: watchpoint.id, + name: watchpoint.name, + description: watchpoint.description ?? '', + check: watchpoint.check, + intervalMs: watchpoint.intervalMs ?? this.config.defaultIntervalMs, + severity: watchpoint.severity ?? AlertSeverity.WARNING, + autoHeal: watchpoint.autoHeal ?? false, + healerId: watchpoint.healerId ?? null, + lastStatus: WatchpointStatus.UNKNOWN, + lastCheck: null, + consecutiveFailures: 0, + }; + + this.watchpoints.set(watchpoint.id, entry); + + // If sentinel is already running, create a timer for the new watchpoint immediately + if (this.running) { + const timer = setInterval(() => { + this.evaluateWatchpoint(entry.id).catch(() => {}); + }, entry.intervalMs); + if (timer.unref) timer.unref(); + this._timers.set(entry.id, timer); + } + + this.emit(Events.WATCHPOINT_REGISTERED, { id: entry.id, name: entry.name }); + return entry; + } + + /** + * Unregister a watchpoint + * @param {string} id - Watchpoint ID + * @returns {boolean} True if removed + */ + unregisterWatchpoint(id) { + if (this._timers.has(id)) { + clearInterval(this._timers.get(id)); + this._timers.delete(id); + } + return this.watchpoints.delete(id); + } + + /** + * Evaluate a single watchpoint + * + * @param {string} id - Watchpoint ID + * @returns {Object} Evaluation result + */ + async evaluateWatchpoint(id) { + const wp = this.watchpoints.get(id); + if (!wp) { + throw new Error(`Unknown watchpoint: ${id}`); + } + + let result; + try { + result = await wp.check(); + } catch (error) { + result = { + status: WatchpointStatus.FAILING, + message: `Check threw: ${error.message}`, + data: null, + }; + } + + const prevStatus = wp.lastStatus; + wp.lastStatus = result.status ?? WatchpointStatus.UNKNOWN; + wp.lastCheck = new Date().toISOString(); + + // Track consecutive failures (DEGRADED counts as a soft failure) + if (wp.lastStatus === WatchpointStatus.FAILING || wp.lastStatus === WatchpointStatus.DEGRADED) { + wp.consecutiveFailures++; + } else if (wp.lastStatus === WatchpointStatus.HEALTHY) { + wp.consecutiveFailures = 0; + } + + const evaluation = { + watchpointId: id, + name: wp.name, + status: wp.lastStatus, + previousStatus: prevStatus, + message: result.message ?? '', + data: result.data ?? null, + consecutiveFailures: wp.consecutiveFailures, + timestamp: wp.lastCheck, + }; + + this.emit(Events.WATCHPOINT_EVALUATED, evaluation); + + // Fire alert on non-healthy status change or repeated failures + if ( + wp.lastStatus !== WatchpointStatus.HEALTHY && + wp.lastStatus !== WatchpointStatus.UNKNOWN && + (prevStatus !== wp.lastStatus || wp.consecutiveFailures > 1) + ) { + this._fireAlert(wp, evaluation); + } + + // Recalculate health score + this._updateHealthScore(); + + return evaluation; + } + + /** + * Evaluate all watchpoints + * @returns {Object[]} All evaluation results + */ + async evaluateAll() { + const results = []; + for (const [id] of this.watchpoints) { + const result = await this.evaluateWatchpoint(id); + results.push(result); + } + return results; + } + + /** + * Start continuous monitoring + */ + start() { + if (this.running) return; + this.running = true; + + for (const [id, wp] of this.watchpoints) { + const timer = setInterval(() => { + this.evaluateWatchpoint(id).catch(() => {}); + }, wp.intervalMs); + + // Prevent timer from keeping process alive + if (timer.unref) timer.unref(); + this._timers.set(id, timer); + } + + this.emit(Events.SENTINEL_STARTED, { + watchpoints: this.watchpoints.size, + timestamp: new Date().toISOString(), + }); + } + + /** + * Stop continuous monitoring + */ + stop() { + if (!this.running) return; + this.running = false; + + for (const [id, timer] of this._timers) { + clearInterval(timer); + this._timers.delete(id); + } + + this.emit(Events.SENTINEL_STOPPED, { + timestamp: new Date().toISOString(), + }); + } + + /** + * Get current health score (0-100) + * @returns {number} Health score + */ + getHealthScore() { + return this._lastHealthScore; + } + + /** + * Get status of all watchpoints + * @returns {Object[]} Watchpoint statuses + */ + getStatus() { + const statuses = []; + for (const [, wp] of this.watchpoints) { + statuses.push({ + id: wp.id, + name: wp.name, + status: wp.lastStatus, + lastCheck: wp.lastCheck, + consecutiveFailures: wp.consecutiveFailures, + severity: wp.severity, + }); + } + return statuses; + } + + /** + * Get alert history + * @param {Object} [filter] + * @param {string} [filter.severity] - Filter by severity + * @param {string} [filter.watchpointId] - Filter by watchpoint + * @param {number} [filter.limit] - Limit results + * @returns {Object[]} Filtered alerts + */ + getAlerts(filter = {}) { + let results = [...this.alerts]; + + if (filter.severity) results = results.filter((a) => a.severity === filter.severity); + if (filter.watchpointId) + results = results.filter((a) => a.watchpointId === filter.watchpointId); + if (filter.limit) results = results.slice(-filter.limit); + + return results; + } + + /** + * Get statistics summary + * @returns {Object} Stats with totals by severity and watchpoint + */ + getStats() { + const bySeverity = {}; + const byWatchpoint = {}; + + for (const alert of this.alerts) { + bySeverity[alert.severity] = (bySeverity[alert.severity] ?? 0) + 1; + byWatchpoint[alert.watchpointId] = (byWatchpoint[alert.watchpointId] ?? 0) + 1; + } + + return { + totalAlerts: this.alerts.length, + totalWatchpoints: this.watchpoints.size, + healthScore: this._lastHealthScore, + running: this.running, + bySeverity, + byWatchpoint, + }; + } + + /** + * Prune old alerts beyond retention window + * @returns {number} Number of pruned alerts + */ + prune() { + const cutoff = Date.now() - this.config.retentionDays * 24 * 60 * 60 * 1000; + const before = this.alerts.length; + + this.alerts = this.alerts.filter((a) => new Date(a.timestamp).getTime() >= cutoff); + + const pruned = before - this.alerts.length; + if (pruned > 0) { + this.emit(Events.ALERTS_PRUNED, { count: pruned }); + } + return pruned; + } + + // ═══════════════════════════════════════════════════════════════════════════════ + // BUILT-IN WATCHPOINTS + // ═══════════════════════════════════════════════════════════════════════════════ + + /** + * Register built-in system watchpoints + */ + registerBuiltInWatchpoints() { + const projectRoot = this.projectRoot; + + // 1. Config file integrity + this.registerWatchpoint({ + id: 'config-integrity', + name: 'Config File Integrity', + description: 'Checks that core JSON config files are valid and YAML files are readable', + severity: AlertSeverity.CRITICAL, + check: async () => { + const configFiles = [ + path.join(projectRoot, '.aiox-core', 'core-config.yaml'), + path.join(projectRoot, 'package.json'), + ]; + + for (const file of configFiles) { + if (!fs.existsSync(file)) continue; + try { + const content = fs.readFileSync(file, 'utf8'); + if (file.endsWith('.json')) { + JSON.parse(content); + } else if (file.endsWith('.yaml') || file.endsWith('.yml')) { + // Without a YAML parser, verify the file is readable and non-empty + if (!content.trim()) { + return { + status: WatchpointStatus.FAILING, + message: `Empty config: ${path.basename(file)}`, + data: { file }, + }; + } + } + } catch { + return { + status: WatchpointStatus.FAILING, + message: `Corrupted config: ${path.basename(file)}`, + data: { file }, + }; + } + } + return { status: WatchpointStatus.HEALTHY, message: 'All configs valid' }; + }, + }); + + // 2. Memory file integrity + this.registerWatchpoint({ + id: 'memory-integrity', + name: 'Memory Files Integrity', + description: 'Checks that .aiox JSON files are parseable', + severity: AlertSeverity.WARNING, + check: async () => { + const memDir = path.join(projectRoot, '.aiox'); + if (!fs.existsSync(memDir)) { + return { status: WatchpointStatus.HEALTHY, message: 'No memory dir yet' }; + } + + const files = fs.readdirSync(memDir).filter((f) => f.endsWith('.json')); + const corrupted = []; + + for (const file of files) { + try { + JSON.parse(fs.readFileSync(path.join(memDir, file), 'utf8')); + } catch { + corrupted.push(file); + } + } + + if (corrupted.length > 0) { + return { + status: WatchpointStatus.FAILING, + message: `Corrupted memory files: ${corrupted.join(', ')}`, + data: { corrupted }, + }; + } + return { status: WatchpointStatus.HEALTHY, message: 'All memory files valid' }; + }, + }); + + // 3. Stale lock files + this.registerWatchpoint({ + id: 'stale-locks', + name: 'Stale Lock File Detection', + description: 'Detects orphaned .lock files that may block operations', + severity: AlertSeverity.WARNING, + autoHeal: true, + check: async () => { + const lockDir = path.join(projectRoot, '.aiox'); + const staleLocks = []; + const maxAge = 30 * 60 * 1000; // 30 minutes + + if (fs.existsSync(lockDir)) { + const files = fs.readdirSync(lockDir).filter((f) => f.endsWith('.lock')); + for (const file of files) { + const filePath = path.join(lockDir, file); + try { + const stat = fs.statSync(filePath); + if (Date.now() - stat.mtimeMs > maxAge) { + staleLocks.push(filePath); + } + } catch { + // ignore inaccessible files + } + } + } + + if (staleLocks.length > 0) { + return { + status: WatchpointStatus.DEGRADED, + message: `Found ${staleLocks.length} stale lock file(s)`, + data: { locks: staleLocks }, + }; + } + return { status: WatchpointStatus.HEALTHY, message: 'No stale locks' }; + }, + }); + + // 4. Disk space (recursive directory size) + this.registerWatchpoint({ + id: 'disk-space', + name: 'Disk Space Monitor', + description: 'Monitors available disk space for .aiox directory', + severity: AlertSeverity.WARNING, + check: async () => { + const memDir = path.join(projectRoot, '.aiox'); + if (!fs.existsSync(memDir)) { + return { status: WatchpointStatus.HEALTHY, message: 'No .aiox dir' }; + } + + const totalSize = this._getDirectorySize(memDir); + + const maxSize = 50 * 1024 * 1024; // 50MB warning threshold + if (totalSize > maxSize) { + return { + status: WatchpointStatus.DEGRADED, + message: `Memory dir is ${Math.round(totalSize / 1024 / 1024)}MB (threshold: 50MB)`, + data: { totalSizeBytes: totalSize }, + }; + } + return { + status: WatchpointStatus.HEALTHY, + message: `Memory dir: ${Math.round(totalSize / 1024)}KB`, + }; + }, + }); + + // 5. Workspace structure + this.registerWatchpoint({ + id: 'workspace-structure', + name: 'Workspace Directory Structure', + description: 'Verifies essential directories exist', + severity: AlertSeverity.INFO, + check: async () => { + const requiredDirs = ['.aiox-core', 'tests', 'bin']; + const missing = requiredDirs.filter((d) => !fs.existsSync(path.join(projectRoot, d))); + + // Also check .aios-core for backward compat + if (missing.includes('.aiox-core') && fs.existsSync(path.join(projectRoot, '.aios-core'))) { + missing.splice(missing.indexOf('.aiox-core'), 1); + } + + if (missing.length > 0) { + return { + status: WatchpointStatus.DEGRADED, + message: `Missing directories: ${missing.join(', ')}`, + data: { missing }, + }; + } + return { status: WatchpointStatus.HEALTHY, message: 'All directories present' }; + }, + }); + } + + // ═══════════════════════════════════════════════════════════════════════════════ + // INTERNAL METHODS + // ═══════════════════════════════════════════════════════════════════════════════ + + /** + * Recursively compute the total size of a directory in bytes + * @param {string} dir - Directory path + * @returns {number} Total size in bytes + * @private + */ + _getDirectorySize(dir) { + let totalSize = 0; + try { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + try { + if (entry.isDirectory()) { + totalSize += this._getDirectorySize(fullPath); + } else { + const stat = fs.statSync(fullPath); + totalSize += stat.size; + } + } catch { + // ignore inaccessible entries + } + } + } catch { + // ignore unreadable directories + } + return totalSize; + } + + /** + * Fire an alert + * @param {Object} watchpoint - The watchpoint that triggered + * @param {Object} evaluation - The evaluation result + * @returns {Object} The fired alert + * @private + */ + _fireAlert(watchpoint, evaluation) { + const alert = { + id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`, + watchpointId: watchpoint.id, + name: watchpoint.name, + severity: watchpoint.severity, + status: evaluation.status, + message: evaluation.message, + data: evaluation.data, + consecutiveFailures: evaluation.consecutiveFailures, + autoHeal: watchpoint.autoHeal, + healerId: watchpoint.healerId, + timestamp: new Date().toISOString(), + }; + + this.alerts.push(alert); + + // Enforce max alerts + if (this.alerts.length > this.config.maxAlerts) { + this.alerts.shift(); + } + + this.emit(Events.ALERT_FIRED, alert); + return alert; + } + + /** + * Update aggregated health score + * @private + */ + _updateHealthScore() { + if (this.watchpoints.size === 0) { + this._lastHealthScore = 100; + return; + } + + let deductions = 0; + for (const [, wp] of this.watchpoints) { + if (wp.lastStatus === WatchpointStatus.FAILING) { + deductions += this.config.healthScoreWeights[wp.severity] ?? 10; + } else if (wp.lastStatus === WatchpointStatus.DEGRADED) { + deductions += (this.config.healthScoreWeights[wp.severity] ?? 10) * 0.5; + } + } + + const newScore = Math.max(0, Math.min(100, 100 - deductions)); + if (newScore !== this._lastHealthScore) { + const prev = this._lastHealthScore; + this._lastHealthScore = newScore; + this.emit(Events.HEALTH_SCORE_CHANGED, { previous: prev, current: newScore }); + } + } +} + +// ═══════════════════════════════════════════════════════════════════════════════════ +// EXPORTS +// ═══════════════════════════════════════════════════════════════════════════════════ + +module.exports = ProactiveSentinel; +module.exports.ProactiveSentinel = ProactiveSentinel; +module.exports.AlertSeverity = AlertSeverity; +module.exports.WatchpointStatus = WatchpointStatus; +module.exports.Events = Events; +module.exports.CONFIG = CONFIG; diff --git a/.aiox-core/core/health-check/proactive-sentinel.js b/.aiox-core/core/health-check/proactive-sentinel.js new file mode 100644 index 0000000000..bb51b147ce --- /dev/null +++ b/.aiox-core/core/health-check/proactive-sentinel.js @@ -0,0 +1,687 @@ +#!/usr/bin/env node + +/** + * AIOX Proactive Sentinel + * + * Story: HCS-3 - Proactive Health Monitoring + * Epic: Epic 9 - Persistent Memory Layer / Health Check System + * + * Continuously monitors system health indicators and triggers + * preventive actions before failures occur. Works with HealerManager + * for automated remediation. + * + * Features: + * - AC1: Registers watchpoints for system health indicators + * - AC2: Evaluates watchpoints on configurable intervals + * - AC3: Fires alerts with severity levels (info, warning, critical) + * - AC4: Tracks alert history for pattern detection + * - AC5: Integrates with HealerManager for auto-remediation + * - AC6: Supports custom watchpoint functions + * - AC7: Provides health score (0-100) aggregated from all watchpoints + * - AC8: Persists alert history in .aiox/sentinel-alerts.json + * + * @author @dev (Dex) + * @version 1.0.0 + */ + +const fs = require('fs'); +const path = require('path'); +const EventEmitter = require('events'); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// CONFIGURATION +// ═══════════════════════════════════════════════════════════════════════════════════ + +const CONFIG = { + alertsPath: '.aiox/sentinel-alerts.json', + maxAlerts: 1000, + retentionDays: 30, + defaultIntervalMs: 60000, // 1 minute + healthScoreWeights: { + critical: 30, + warning: 10, + info: 2, + }, + version: '1.0.0', + schemaVersion: 'aiox-sentinel-v1', +}; + +// ═══════════════════════════════════════════════════════════════════════════════════ +// ENUMS +// ═══════════════════════════════════════════════════════════════════════════════════ + +const AlertSeverity = { + INFO: 'info', + WARNING: 'warning', + CRITICAL: 'critical', +}; + +const WatchpointStatus = { + HEALTHY: 'healthy', + DEGRADED: 'degraded', + FAILING: 'failing', + UNKNOWN: 'unknown', +}; + +const Events = { + ALERT_FIRED: 'sentinel:alert', + WATCHPOINT_REGISTERED: 'sentinel:watchpoint:registered', + WATCHPOINT_EVALUATED: 'sentinel:watchpoint:evaluated', + HEALTH_SCORE_CHANGED: 'sentinel:health:changed', + SENTINEL_STARTED: 'sentinel:started', + SENTINEL_STOPPED: 'sentinel:stopped', + ALERTS_PRUNED: 'sentinel:alerts:pruned', +}; + +// ═══════════════════════════════════════════════════════════════════════════════════ +// PROACTIVE SENTINEL +// ═══════════════════════════════════════════════════════════════════════════════════ + +class ProactiveSentinel extends EventEmitter { + constructor(options = {}) { + super(); + this.projectRoot = options.projectRoot || process.cwd(); + this.config = { ...CONFIG, ...options.config }; + this.watchpoints = new Map(); + this.alerts = []; + this.running = false; + this._timers = new Map(); + this._lastHealthScore = 100; + this._loaded = false; + } + + /** + * Get the alerts file path + */ + _getFilePath() { + return path.join(this.projectRoot, this.config.alertsPath); + } + + /** + * Load alert history from disk + */ + async load() { + const filePath = this._getFilePath(); + try { + if (fs.existsSync(filePath)) { + const raw = fs.readFileSync(filePath, 'utf8'); + const data = JSON.parse(raw); + + if (data.schemaVersion !== this.config.schemaVersion) { + this.alerts = []; + this._loaded = true; + return; + } + + this.alerts = Array.isArray(data.alerts) ? data.alerts : []; + } + } catch { + this.alerts = []; + } + this._loaded = true; + } + + /** + * Save alert history to disk + */ + async save() { + const filePath = this._getFilePath(); + const dir = path.dirname(filePath); + + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const data = { + schemaVersion: this.config.schemaVersion, + version: this.config.version, + savedAt: new Date().toISOString(), + alerts: this.alerts, + }; + + fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8'); + } + + /** + * Register a watchpoint + * + * @param {Object} watchpoint + * @param {string} watchpoint.id - Unique watchpoint ID + * @param {string} watchpoint.name - Human-readable name + * @param {string} watchpoint.description - What this monitors + * @param {Function} watchpoint.check - Async function returning { status, message, data } + * @param {number} [watchpoint.intervalMs] - Check interval (default: 60000) + * @param {string} [watchpoint.severity] - Alert severity when failing (default: warning) + * @param {boolean} [watchpoint.autoHeal] - Whether to trigger auto-healing (default: false) + * @param {string} [watchpoint.healerId] - HealerManager check ID for remediation + */ + registerWatchpoint(watchpoint) { + if (!watchpoint.id || !watchpoint.name || !watchpoint.check) { + throw new Error('Required fields: id, name, check'); + } + + if (typeof watchpoint.check !== 'function') { + throw new Error('check must be a function'); + } + + const entry = { + id: watchpoint.id, + name: watchpoint.name, + description: watchpoint.description || '', + check: watchpoint.check, + intervalMs: watchpoint.intervalMs ?? this.config.defaultIntervalMs, + severity: watchpoint.severity ?? AlertSeverity.WARNING, + autoHeal: watchpoint.autoHeal ?? false, + healerId: watchpoint.healerId || null, + lastStatus: WatchpointStatus.UNKNOWN, + lastCheck: null, + consecutiveFailures: 0, + }; + + this.watchpoints.set(watchpoint.id, entry); + + // If sentinel is already running, create a timer for the new watchpoint immediately + if (this.running) { + const timer = setInterval(() => { + this.evaluateWatchpoint(entry.id).catch(() => {}); + }, entry.intervalMs); + if (timer.unref) timer.unref(); + this._timers.set(entry.id, timer); + } + + this.emit(Events.WATCHPOINT_REGISTERED, { id: entry.id, name: entry.name }); + return entry; + } + + /** + * Unregister a watchpoint + * @param {string} id - Watchpoint ID + * @returns {boolean} True if removed + */ + unregisterWatchpoint(id) { + if (this._timers.has(id)) { + clearInterval(this._timers.get(id)); + this._timers.delete(id); + } + return this.watchpoints.delete(id); + } + + /** + * Evaluate a single watchpoint + * + * @param {string} id - Watchpoint ID + * @returns {Object} Evaluation result + */ + async evaluateWatchpoint(id) { + const wp = this.watchpoints.get(id); + if (!wp) { + throw new Error(`Unknown watchpoint: ${id}`); + } + + let result; + try { + result = await wp.check(); + } catch (error) { + result = { + status: WatchpointStatus.FAILING, + message: `Check threw: ${error.message}`, + data: null, + }; + } + + const prevStatus = wp.lastStatus; + wp.lastStatus = result.status || WatchpointStatus.UNKNOWN; + wp.lastCheck = new Date().toISOString(); + + // Track consecutive failures (DEGRADED counts as a soft failure) + if (wp.lastStatus === WatchpointStatus.FAILING || wp.lastStatus === WatchpointStatus.DEGRADED) { + wp.consecutiveFailures++; + } else if (wp.lastStatus === WatchpointStatus.HEALTHY) { + wp.consecutiveFailures = 0; + } + + const evaluation = { + watchpointId: id, + name: wp.name, + status: wp.lastStatus, + previousStatus: prevStatus, + message: result.message || '', + data: result.data || null, + consecutiveFailures: wp.consecutiveFailures, + timestamp: wp.lastCheck, + }; + + this.emit(Events.WATCHPOINT_EVALUATED, evaluation); + + // Fire alert on non-healthy status or repeated failures + if ( + wp.lastStatus !== WatchpointStatus.HEALTHY && + wp.lastStatus !== WatchpointStatus.UNKNOWN && + (prevStatus !== wp.lastStatus || wp.consecutiveFailures > 1) + ) { + this._fireAlert(wp, evaluation); + } + + // Recalculate health score + this._updateHealthScore(); + + return evaluation; + } + + /** + * Evaluate all watchpoints + * @returns {Object[]} All evaluation results + */ + async evaluateAll() { + const results = []; + for (const [id] of this.watchpoints) { + const result = await this.evaluateWatchpoint(id); + results.push(result); + } + return results; + } + + /** + * Start continuous monitoring + */ + start() { + if (this.running) return; + this.running = true; + + for (const [id, wp] of this.watchpoints) { + const timer = setInterval(() => { + this.evaluateWatchpoint(id).catch(() => {}); + }, wp.intervalMs); + + // Prevent timer from keeping process alive + if (timer.unref) timer.unref(); + this._timers.set(id, timer); + } + + this.emit(Events.SENTINEL_STARTED, { + watchpoints: this.watchpoints.size, + timestamp: new Date().toISOString(), + }); + } + + /** + * Stop continuous monitoring + */ + stop() { + if (!this.running) return; + this.running = false; + + for (const [id, timer] of this._timers) { + clearInterval(timer); + this._timers.delete(id); + } + + this.emit(Events.SENTINEL_STOPPED, { + timestamp: new Date().toISOString(), + }); + } + + /** + * Get current health score (0-100) + * @returns {number} Health score + */ + getHealthScore() { + return this._lastHealthScore; + } + + /** + * Get status of all watchpoints + * @returns {Object[]} Watchpoint statuses + */ + getStatus() { + const statuses = []; + for (const [, wp] of this.watchpoints) { + statuses.push({ + id: wp.id, + name: wp.name, + status: wp.lastStatus, + lastCheck: wp.lastCheck, + consecutiveFailures: wp.consecutiveFailures, + severity: wp.severity, + }); + } + return statuses; + } + + /** + * Get alert history + * @param {Object} [filter] + * @param {string} [filter.severity] + * @param {string} [filter.watchpointId] + * @param {number} [filter.limit] + * @returns {Object[]} Filtered alerts + */ + getAlerts(filter = {}) { + let results = [...this.alerts]; + + if (filter.severity) results = results.filter((a) => a.severity === filter.severity); + if (filter.watchpointId) + results = results.filter((a) => a.watchpointId === filter.watchpointId); + if (filter.limit) results = results.slice(-filter.limit); + + return results; + } + + /** + * Get statistics summary + */ + getStats() { + const bySeverity = {}; + const byWatchpoint = {}; + + for (const alert of this.alerts) { + bySeverity[alert.severity] = (bySeverity[alert.severity] || 0) + 1; + byWatchpoint[alert.watchpointId] = (byWatchpoint[alert.watchpointId] || 0) + 1; + } + + return { + totalAlerts: this.alerts.length, + totalWatchpoints: this.watchpoints.size, + healthScore: this._lastHealthScore, + running: this.running, + bySeverity, + byWatchpoint, + }; + } + + /** + * Prune old alerts beyond retention window + * @returns {number} Number of pruned alerts + */ + prune() { + const cutoff = Date.now() - this.config.retentionDays * 24 * 60 * 60 * 1000; + const before = this.alerts.length; + + this.alerts = this.alerts.filter((a) => new Date(a.timestamp).getTime() >= cutoff); + + const pruned = before - this.alerts.length; + if (pruned > 0) { + this.emit(Events.ALERTS_PRUNED, { count: pruned }); + } + return pruned; + } + + // ═══════════════════════════════════════════════════════════════════════════════ + // BUILT-IN WATCHPOINTS + // ═══════════════════════════════════════════════════════════════════════════════ + + /** + * Register built-in system watchpoints + */ + registerBuiltInWatchpoints() { + const projectRoot = this.projectRoot; + + // 1. Config file integrity + this.registerWatchpoint({ + id: 'config-integrity', + name: 'Config File Integrity', + description: 'Checks that core JSON config files are valid and YAML files are readable', + severity: AlertSeverity.CRITICAL, + check: async () => { + const configFiles = [ + path.join(projectRoot, 'core-config.yaml'), + path.join(projectRoot, 'package.json'), + ]; + + for (const file of configFiles) { + if (!fs.existsSync(file)) continue; + try { + const content = fs.readFileSync(file, 'utf8'); + if (file.endsWith('.json')) { + JSON.parse(content); + } else if (file.endsWith('.yaml') || file.endsWith('.yml')) { + // Without a YAML parser, verify the file is readable and non-empty + if (!content.trim()) { + return { + status: WatchpointStatus.FAILING, + message: `Empty config: ${path.basename(file)}`, + data: { file }, + }; + } + } + } catch { + return { + status: WatchpointStatus.FAILING, + message: `Corrupted config: ${path.basename(file)}`, + data: { file }, + }; + } + } + return { status: WatchpointStatus.HEALTHY, message: 'All configs valid' }; + }, + }); + + // 2. Memory file integrity + this.registerWatchpoint({ + id: 'memory-integrity', + name: 'Memory Files Integrity', + description: 'Checks that .aiox JSON files are parseable', + severity: AlertSeverity.WARNING, + check: async () => { + const memDir = path.join(projectRoot, '.aiox'); + if (!fs.existsSync(memDir)) { + return { status: WatchpointStatus.HEALTHY, message: 'No memory dir yet' }; + } + + const files = fs.readdirSync(memDir).filter((f) => f.endsWith('.json')); + const corrupted = []; + + for (const file of files) { + try { + JSON.parse(fs.readFileSync(path.join(memDir, file), 'utf8')); + } catch { + corrupted.push(file); + } + } + + if (corrupted.length > 0) { + return { + status: WatchpointStatus.FAILING, + message: `Corrupted memory files: ${corrupted.join(', ')}`, + data: { corrupted }, + }; + } + return { status: WatchpointStatus.HEALTHY, message: 'All memory files valid' }; + }, + }); + + // 3. Stale lock files + this.registerWatchpoint({ + id: 'stale-locks', + name: 'Stale Lock File Detection', + description: 'Detects orphaned .lock files that may block operations', + severity: AlertSeverity.WARNING, + autoHeal: true, + check: async () => { + const lockDir = path.join(projectRoot, '.aiox'); + const staleLocks = []; + const maxAge = 30 * 60 * 1000; // 30 minutes + + if (fs.existsSync(lockDir)) { + const files = fs.readdirSync(lockDir).filter((f) => f.endsWith('.lock')); + for (const file of files) { + const filePath = path.join(lockDir, file); + try { + const stat = fs.statSync(filePath); + if (Date.now() - stat.mtimeMs > maxAge) { + staleLocks.push(filePath); + } + } catch { + // ignore inaccessible files + } + } + } + + if (staleLocks.length > 0) { + return { + status: WatchpointStatus.DEGRADED, + message: `Found ${staleLocks.length} stale lock file(s)`, + data: { locks: staleLocks }, + }; + } + return { status: WatchpointStatus.HEALTHY, message: 'No stale locks' }; + }, + }); + + // 4. Disk space + this.registerWatchpoint({ + id: 'disk-space', + name: 'Disk Space Monitor', + description: 'Monitors available disk space for .aiox directory', + severity: AlertSeverity.WARNING, + check: async () => { + const memDir = path.join(projectRoot, '.aiox'); + if (!fs.existsSync(memDir)) { + return { status: WatchpointStatus.HEALTHY, message: 'No .aiox dir' }; + } + + const totalSize = this._getDirectorySize(memDir); + + const maxSize = 50 * 1024 * 1024; // 50MB warning threshold + if (totalSize > maxSize) { + return { + status: WatchpointStatus.DEGRADED, + message: `Memory dir is ${Math.round(totalSize / 1024 / 1024)}MB (threshold: 50MB)`, + data: { totalSizeBytes: totalSize }, + }; + } + return { + status: WatchpointStatus.HEALTHY, + message: `Memory dir: ${Math.round(totalSize / 1024)}KB`, + }; + }, + }); + + // 5. Workspace structure + this.registerWatchpoint({ + id: 'workspace-structure', + name: 'Workspace Directory Structure', + description: 'Verifies essential directories exist', + severity: AlertSeverity.INFO, + check: async () => { + const requiredDirs = ['.aiox-core', 'tests', 'bin']; + const missing = requiredDirs.filter((d) => !fs.existsSync(path.join(projectRoot, d))); + + // Also check .aios-core for backward compat + if (missing.includes('.aiox-core') && fs.existsSync(path.join(projectRoot, '.aios-core'))) { + missing.splice(missing.indexOf('.aiox-core'), 1); + } + + if (missing.length > 0) { + return { + status: WatchpointStatus.DEGRADED, + message: `Missing directories: ${missing.join(', ')}`, + data: { missing }, + }; + } + return { status: WatchpointStatus.HEALTHY, message: 'All directories present' }; + }, + }); + } + + // ═══════════════════════════════════════════════════════════════════════════════ + // INTERNAL METHODS + // ═══════════════════════════════════════════════════════════════════════════════ + + /** + * Recursively compute the total size of a directory in bytes + * @param {string} dir - Directory path + * @returns {number} Total size in bytes + * @private + */ + _getDirectorySize(dir) { + let totalSize = 0; + try { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + try { + if (entry.isDirectory()) { + totalSize += this._getDirectorySize(fullPath); + } else { + const stat = fs.statSync(fullPath); + totalSize += stat.size; + } + } catch { + // ignore inaccessible entries + } + } + } catch { + // ignore unreadable directories + } + return totalSize; + } + + /** + * Fire an alert + * @private + */ + _fireAlert(watchpoint, evaluation) { + const alert = { + id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`, + watchpointId: watchpoint.id, + name: watchpoint.name, + severity: watchpoint.severity, + status: evaluation.status, + message: evaluation.message, + data: evaluation.data, + consecutiveFailures: evaluation.consecutiveFailures, + autoHeal: watchpoint.autoHeal, + healerId: watchpoint.healerId, + timestamp: new Date().toISOString(), + }; + + this.alerts.push(alert); + + // Enforce max alerts + if (this.alerts.length > this.config.maxAlerts) { + this.alerts.shift(); + } + + this.emit(Events.ALERT_FIRED, alert); + return alert; + } + + /** + * Update aggregated health score + * @private + */ + _updateHealthScore() { + if (this.watchpoints.size === 0) { + this._lastHealthScore = 100; + return; + } + + let deductions = 0; + for (const [, wp] of this.watchpoints) { + if (wp.lastStatus === WatchpointStatus.FAILING) { + deductions += this.config.healthScoreWeights[wp.severity] || 10; + } else if (wp.lastStatus === WatchpointStatus.DEGRADED) { + deductions += (this.config.healthScoreWeights[wp.severity] || 10) * 0.5; + } + } + + const newScore = Math.max(0, Math.min(100, 100 - deductions)); + if (newScore !== this._lastHealthScore) { + const prev = this._lastHealthScore; + this._lastHealthScore = newScore; + this.emit(Events.HEALTH_SCORE_CHANGED, { previous: prev, current: newScore }); + } + } +} + +// ═══════════════════════════════════════════════════════════════════════════════════ +// EXPORTS +// ═══════════════════════════════════════════════════════════════════════════════════ + +module.exports = ProactiveSentinel; +module.exports.ProactiveSentinel = ProactiveSentinel; +module.exports.AlertSeverity = AlertSeverity; +module.exports.WatchpointStatus = WatchpointStatus; +module.exports.Events = Events; +module.exports.CONFIG = CONFIG; diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index 05816fa4fd..2fb1ca8cf8 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,9 +8,9 @@ # - File types for categorization # version: 5.0.3 -generated_at: "2026-03-11T15:04:09.395Z" +generated_at: "2026-03-18T02:02:39.902Z" generator: scripts/generate-install-manifest.js -file_count: 1090 +file_count: 1091 files: - path: cli/commands/config/index.js hash: sha256:25c4b9bf4e0241abf7754b55153f49f1a214f1fb5fe904a576675634cb7b3da9 @@ -672,6 +672,10 @@ files: hash: sha256:738bec37c11cfb02e9df96e3631f74fa0652f5a531c7b0f1d8887e0141b1f72e type: core size: 10726 + - path: core/health-check/proactive-sentinel.js + hash: sha256:cbaa0425e705075d4e92fd77e7e243c5662b4096b70f7760a59aab85681881ff + type: core + size: 22434 - path: core/health-check/reporters/console.js hash: sha256:573f28a6c9c2a4087ccd349398f47351aa9a752c92a1f2e4a3c3f396682d5516 type: core diff --git a/tests/core/health-check/proactive-sentinel.test.js b/tests/core/health-check/proactive-sentinel.test.js new file mode 100644 index 0000000000..08744fee11 --- /dev/null +++ b/tests/core/health-check/proactive-sentinel.test.js @@ -0,0 +1,736 @@ +/** + * Tests for Proactive Sentinel + * + * Story: HCS-3 - Proactive Health Monitoring + */ + +const fs = require('fs'); +const path = require('path'); +const ProactiveSentinel = require('../../../.aios-core/core/health-check/proactive-sentinel'); +const { AlertSeverity, WatchpointStatus, Events, CONFIG } = ProactiveSentinel; + +// ═══════════════════════════════════════════════════════════════════════════════════ +// TEST SETUP +// ═══════════════════════════════════════════════════════════════════════════════════ + +const TEST_DIR = path.join(__dirname, '__test-sentinel__'); + +function createSentinel(overrides = {}) { + return new ProactiveSentinel({ + projectRoot: TEST_DIR, + config: { + alertsPath: '.aiox/sentinel-alerts.json', + ...overrides, + }, + }); +} + +function healthyCheck() { + return async () => ({ + status: WatchpointStatus.HEALTHY, + message: 'All good', + }); +} + +function failingCheck(msg = 'Something broke') { + return async () => ({ + status: WatchpointStatus.FAILING, + message: msg, + data: { detail: 'error' }, + }); +} + +function degradedCheck() { + return async () => ({ + status: WatchpointStatus.DEGRADED, + message: 'Partially working', + }); +} + +beforeEach(() => { + if (fs.existsSync(TEST_DIR)) { + fs.rmSync(TEST_DIR, { recursive: true, force: true }); + } + fs.mkdirSync(TEST_DIR, { recursive: true }); +}); + +afterAll(() => { + if (fs.existsSync(TEST_DIR)) { + fs.rmSync(TEST_DIR, { recursive: true, force: true }); + } +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// CONSTRUCTOR +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - constructor', () => { + test('initializes with defaults', () => { + const s = createSentinel(); + expect(s.watchpoints.size).toBe(0); + expect(s.alerts).toEqual([]); + expect(s.running).toBe(false); + expect(s._lastHealthScore).toBe(100); + }); + + test('accepts custom config', () => { + const s = createSentinel({ maxAlerts: 50 }); + expect(s.config.maxAlerts).toBe(50); + }); + + test('is an EventEmitter', () => { + const s = createSentinel(); + expect(typeof s.on).toBe('function'); + expect(typeof s.emit).toBe('function'); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// LOAD / SAVE +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - load/save', () => { + test('loads from empty state', async () => { + const s = createSentinel(); + await s.load(); + expect(s._loaded).toBe(true); + expect(s.alerts).toEqual([]); + }); + + test('save creates directory and file', async () => { + const s = createSentinel(); + await s.load(); + s.alerts.push({ id: 'test', timestamp: new Date().toISOString() }); + await s.save(); + + const filePath = path.join(TEST_DIR, '.aiox/sentinel-alerts.json'); + expect(fs.existsSync(filePath)).toBe(true); + }); + + test('round-trip: save then load', async () => { + const s1 = createSentinel(); + await s1.load(); + s1.alerts.push({ + id: 'a1', + watchpointId: 'test', + severity: AlertSeverity.WARNING, + timestamp: new Date().toISOString(), + }); + await s1.save(); + + const s2 = createSentinel(); + await s2.load(); + expect(s2.alerts).toHaveLength(1); + expect(s2.alerts[0].id).toBe('a1'); + }); + + test('handles corrupted file', async () => { + const dir = path.join(TEST_DIR, '.aiox'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'sentinel-alerts.json'), 'BROKEN'); + + const s = createSentinel(); + await s.load(); + expect(s._loaded).toBe(true); + expect(s.alerts).toEqual([]); + }); + + test('resets on schema mismatch', async () => { + const dir = path.join(TEST_DIR, '.aiox'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'sentinel-alerts.json'), + JSON.stringify({ schemaVersion: 'old-v0', alerts: [{ id: 'old' }] }), + ); + + const s = createSentinel(); + await s.load(); + expect(s.alerts).toEqual([]); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// REGISTER WATCHPOINT +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - registerWatchpoint', () => { + test('registers a watchpoint', () => { + const s = createSentinel(); + const wp = s.registerWatchpoint({ + id: 'test-wp', + name: 'Test Watchpoint', + description: 'Monitors something', + check: healthyCheck(), + }); + + expect(wp.id).toBe('test-wp'); + expect(s.watchpoints.size).toBe(1); + expect(s.watchpoints.get('test-wp').lastStatus).toBe(WatchpointStatus.UNKNOWN); + }); + + test('throws on missing required fields', () => { + const s = createSentinel(); + expect(() => s.registerWatchpoint({ id: 'x' })).toThrow('Required fields'); + expect(() => s.registerWatchpoint({})).toThrow('Required fields'); + }); + + test('throws when check is not a function', () => { + const s = createSentinel(); + expect(() => + s.registerWatchpoint({ id: 'x', name: 'X', check: 'not-a-function' }), + ).toThrow('check must be a function'); + }); + + test('emits WATCHPOINT_REGISTERED event', () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.WATCHPOINT_REGISTERED, handler); + + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + expect(handler).toHaveBeenCalledWith({ id: 'wp1', name: 'WP1' }); + }); + + test('uses default values for optional fields', () => { + const s = createSentinel(); + const wp = s.registerWatchpoint({ id: 'wp2', name: 'WP2', check: healthyCheck() }); + + expect(wp.severity).toBe(AlertSeverity.WARNING); + expect(wp.autoHeal).toBe(false); + expect(wp.healerId).toBeNull(); + expect(wp.intervalMs).toBe(CONFIG.defaultIntervalMs); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// UNREGISTER WATCHPOINT +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - unregisterWatchpoint', () => { + test('removes a watchpoint', () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + expect(s.unregisterWatchpoint('wp1')).toBe(true); + expect(s.watchpoints.size).toBe(0); + }); + + test('returns false for unknown watchpoint', () => { + const s = createSentinel(); + expect(s.unregisterWatchpoint('nonexistent')).toBe(false); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// EVALUATE WATCHPOINT +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - evaluateWatchpoint', () => { + test('evaluates a healthy watchpoint', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + + const result = await s.evaluateWatchpoint('wp1'); + expect(result.status).toBe(WatchpointStatus.HEALTHY); + expect(result.watchpointId).toBe('wp1'); + expect(result.consecutiveFailures).toBe(0); + }); + + test('evaluates a failing watchpoint', async () => { + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: failingCheck('DB down'), + severity: AlertSeverity.CRITICAL, + }); + + const result = await s.evaluateWatchpoint('wp1'); + expect(result.status).toBe(WatchpointStatus.FAILING); + expect(result.message).toBe('DB down'); + expect(result.consecutiveFailures).toBe(1); + }); + + test('tracks consecutive failures', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: failingCheck() }); + + await s.evaluateWatchpoint('wp1'); + await s.evaluateWatchpoint('wp1'); + const r = await s.evaluateWatchpoint('wp1'); + expect(r.consecutiveFailures).toBe(3); + }); + + test('resets consecutive failures on healthy', async () => { + let shouldFail = true; + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: async () => ({ + status: shouldFail ? WatchpointStatus.FAILING : WatchpointStatus.HEALTHY, + message: shouldFail ? 'fail' : 'ok', + }), + }); + + await s.evaluateWatchpoint('wp1'); + await s.evaluateWatchpoint('wp1'); + shouldFail = false; + const r = await s.evaluateWatchpoint('wp1'); + expect(r.consecutiveFailures).toBe(0); + }); + + test('throws for unknown watchpoint', async () => { + const s = createSentinel(); + await expect(s.evaluateWatchpoint('missing')).rejects.toThrow('Unknown watchpoint'); + }); + + test('handles check function that throws', async () => { + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: async () => { + throw new Error('Boom'); + }, + }); + + const result = await s.evaluateWatchpoint('wp1'); + expect(result.status).toBe(WatchpointStatus.FAILING); + expect(result.message).toContain('Boom'); + }); + + test('emits WATCHPOINT_EVALUATED event', async () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.WATCHPOINT_EVALUATED, handler); + + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + await s.evaluateWatchpoint('wp1'); + + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ watchpointId: 'wp1', status: WatchpointStatus.HEALTHY }), + ); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// EVALUATE ALL +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - evaluateAll', () => { + test('evaluates all watchpoints', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + s.registerWatchpoint({ id: 'wp2', name: 'WP2', check: failingCheck() }); + + const results = await s.evaluateAll(); + expect(results).toHaveLength(2); + expect(results[0].status).toBe(WatchpointStatus.HEALTHY); + expect(results[1].status).toBe(WatchpointStatus.FAILING); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// ALERTS +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - alerts', () => { + test('fires alert on failing watchpoint', async () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.ALERT_FIRED, handler); + + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: failingCheck(), + severity: AlertSeverity.CRITICAL, + }); + + await s.evaluateWatchpoint('wp1'); + + expect(handler).toHaveBeenCalledTimes(1); + const alert = handler.mock.calls[0][0]; + expect(alert.severity).toBe(AlertSeverity.CRITICAL); + expect(alert.watchpointId).toBe('wp1'); + }); + + test('does not fire alert for healthy watchpoint', async () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.ALERT_FIRED, handler); + + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + await s.evaluateWatchpoint('wp1'); + + expect(handler).not.toHaveBeenCalled(); + }); + + test('records alert in history', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: failingCheck() }); + + await s.evaluateWatchpoint('wp1'); + expect(s.alerts).toHaveLength(1); + expect(s.alerts[0].watchpointId).toBe('wp1'); + }); + + test('enforces maxAlerts limit', async () => { + const s = createSentinel({ maxAlerts: 2 }); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: failingCheck() }); + + await s.evaluateWatchpoint('wp1'); + await s.evaluateWatchpoint('wp1'); + await s.evaluateWatchpoint('wp1'); + + expect(s.alerts.length).toBeLessThanOrEqual(2); + }); + + test('getAlerts filters by severity', async () => { + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: failingCheck(), + severity: AlertSeverity.CRITICAL, + }); + s.registerWatchpoint({ + id: 'wp2', + name: 'WP2', + check: failingCheck(), + severity: AlertSeverity.WARNING, + }); + + await s.evaluateWatchpoint('wp1'); + await s.evaluateWatchpoint('wp2'); + + const criticals = s.getAlerts({ severity: AlertSeverity.CRITICAL }); + expect(criticals).toHaveLength(1); + expect(criticals[0].watchpointId).toBe('wp1'); + }); + + test('getAlerts filters by watchpointId', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: failingCheck() }); + s.registerWatchpoint({ id: 'wp2', name: 'WP2', check: failingCheck() }); + + await s.evaluateWatchpoint('wp1'); + await s.evaluateWatchpoint('wp2'); + + expect(s.getAlerts({ watchpointId: 'wp2' })).toHaveLength(1); + }); + + test('getAlerts applies limit', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: failingCheck() }); + + for (let i = 0; i < 5; i++) { + await s.evaluateWatchpoint('wp1'); + } + + expect(s.getAlerts({ limit: 2 })).toHaveLength(2); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// HEALTH SCORE +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - health score', () => { + test('starts at 100', () => { + const s = createSentinel(); + expect(s.getHealthScore()).toBe(100); + }); + + test('decreases on failing watchpoint', async () => { + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: failingCheck(), + severity: AlertSeverity.CRITICAL, + }); + + await s.evaluateWatchpoint('wp1'); + expect(s.getHealthScore()).toBeLessThan(100); + }); + + test('partially decreases on degraded watchpoint', async () => { + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: degradedCheck(), + severity: AlertSeverity.WARNING, + }); + + await s.evaluateWatchpoint('wp1'); + const score = s.getHealthScore(); + expect(score).toBeLessThan(100); + expect(score).toBeGreaterThan(85); // Should only be a minor deduction + }); + + test('emits HEALTH_SCORE_CHANGED event', async () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.HEALTH_SCORE_CHANGED, handler); + + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: failingCheck(), + severity: AlertSeverity.CRITICAL, + }); + + await s.evaluateWatchpoint('wp1'); + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ previous: 100 }), + ); + }); + + test('recovers when watchpoint becomes healthy', async () => { + let healthy = false; + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: async () => ({ + status: healthy ? WatchpointStatus.HEALTHY : WatchpointStatus.FAILING, + message: healthy ? 'ok' : 'fail', + }), + severity: AlertSeverity.CRITICAL, + }); + + await s.evaluateWatchpoint('wp1'); + const lowScore = s.getHealthScore(); + + healthy = true; + await s.evaluateWatchpoint('wp1'); + expect(s.getHealthScore()).toBeGreaterThan(lowScore); + }); + + test('never goes below 0', async () => { + const s = createSentinel(); + // Register many failing critical watchpoints + for (let i = 0; i < 10; i++) { + s.registerWatchpoint({ + id: `wp-${i}`, + name: `WP ${i}`, + check: failingCheck(), + severity: AlertSeverity.CRITICAL, + }); + } + + await s.evaluateAll(); + expect(s.getHealthScore()).toBeGreaterThanOrEqual(0); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// START / STOP +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - start/stop', () => { + test('emits SENTINEL_STARTED on start', () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.SENTINEL_STARTED, handler); + + s.start(); + expect(handler).toHaveBeenCalledTimes(1); + expect(s.running).toBe(true); + s.stop(); + }); + + test('emits SENTINEL_STOPPED on stop', () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.SENTINEL_STOPPED, handler); + + s.start(); + s.stop(); + expect(handler).toHaveBeenCalledTimes(1); + expect(s.running).toBe(false); + }); + + test('does not double-start', () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.SENTINEL_STARTED, handler); + + s.start(); + s.start(); + expect(handler).toHaveBeenCalledTimes(1); + s.stop(); + }); + + test('does not double-stop', () => { + const s = createSentinel(); + const handler = jest.fn(); + s.on(Events.SENTINEL_STOPPED, handler); + + s.stop(); // not running + expect(handler).not.toHaveBeenCalled(); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// STATUS & STATS +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - getStatus', () => { + test('returns status of all watchpoints', async () => { + const s = createSentinel(); + s.registerWatchpoint({ id: 'wp1', name: 'WP1', check: healthyCheck() }); + s.registerWatchpoint({ id: 'wp2', name: 'WP2', check: failingCheck() }); + + await s.evaluateAll(); + const status = s.getStatus(); + + expect(status).toHaveLength(2); + expect(status[0].status).toBe(WatchpointStatus.HEALTHY); + expect(status[1].status).toBe(WatchpointStatus.FAILING); + }); +}); + +describe('ProactiveSentinel - getStats', () => { + test('returns complete statistics', async () => { + const s = createSentinel(); + s.registerWatchpoint({ + id: 'wp1', + name: 'WP1', + check: failingCheck(), + severity: AlertSeverity.CRITICAL, + }); + + await s.evaluateWatchpoint('wp1'); + + const stats = s.getStats(); + expect(stats.totalAlerts).toBe(1); + expect(stats.totalWatchpoints).toBe(1); + expect(stats.running).toBe(false); + expect(stats.bySeverity[AlertSeverity.CRITICAL]).toBe(1); + expect(stats.byWatchpoint['wp1']).toBe(1); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// PRUNE +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - prune', () => { + test('removes old alerts', () => { + const s = createSentinel({ retentionDays: 7 }); + s.alerts.push({ + id: 'old', + watchpointId: 'wp1', + timestamp: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000).toISOString(), + }); + s.alerts.push({ + id: 'recent', + watchpointId: 'wp1', + timestamp: new Date().toISOString(), + }); + + const pruned = s.prune(); + expect(pruned).toBe(1); + expect(s.alerts).toHaveLength(1); + expect(s.alerts[0].id).toBe('recent'); + }); + + test('emits ALERTS_PRUNED event', () => { + const s = createSentinel({ retentionDays: 1 }); + const handler = jest.fn(); + s.on(Events.ALERTS_PRUNED, handler); + + s.alerts.push({ + id: 'old', + watchpointId: 'wp1', + timestamp: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(), + }); + + s.prune(); + expect(handler).toHaveBeenCalledWith({ count: 1 }); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════════ +// BUILT-IN WATCHPOINTS +// ═══════════════════════════════════════════════════════════════════════════════════ + +describe('ProactiveSentinel - registerBuiltInWatchpoints', () => { + test('registers all built-in watchpoints', () => { + const s = createSentinel(); + s.registerBuiltInWatchpoints(); + + expect(s.watchpoints.size).toBe(5); + expect(s.watchpoints.has('config-integrity')).toBe(true); + expect(s.watchpoints.has('memory-integrity')).toBe(true); + expect(s.watchpoints.has('stale-locks')).toBe(true); + expect(s.watchpoints.has('disk-space')).toBe(true); + expect(s.watchpoints.has('workspace-structure')).toBe(true); + }); + + test('config-integrity passes with valid package.json', async () => { + fs.writeFileSync(path.join(TEST_DIR, 'package.json'), JSON.stringify({ name: 'test' })); + fs.writeFileSync(path.join(TEST_DIR, 'core-config.yaml'), 'key: value'); + + const s = createSentinel(); + s.registerBuiltInWatchpoints(); + + const result = await s.evaluateWatchpoint('config-integrity'); + expect(result.status).toBe(WatchpointStatus.HEALTHY); + }); + + test('config-integrity fails with corrupted package.json', async () => { + fs.writeFileSync(path.join(TEST_DIR, 'package.json'), 'NOT JSON!!!'); + + const s = createSentinel(); + s.registerBuiltInWatchpoints(); + + const result = await s.evaluateWatchpoint('config-integrity'); + expect(result.status).toBe(WatchpointStatus.FAILING); + }); + + test('memory-integrity detects corrupted JSON files', async () => { + const memDir = path.join(TEST_DIR, '.aiox'); + fs.mkdirSync(memDir, { recursive: true }); + fs.writeFileSync(path.join(memDir, 'gotchas.json'), 'BROKEN'); + + const s = createSentinel(); + s.registerBuiltInWatchpoints(); + + const result = await s.evaluateWatchpoint('memory-integrity'); + expect(result.status).toBe(WatchpointStatus.FAILING); + expect(result.message).toContain('gotchas.json'); + }); + + test('stale-locks detects old lock files', async () => { + const memDir = path.join(TEST_DIR, '.aiox'); + fs.mkdirSync(memDir, { recursive: true }); + const lockFile = path.join(memDir, 'test.lock'); + fs.writeFileSync(lockFile, 'locked'); + + // Set mtime to 31 minutes ago + const oldTime = new Date(Date.now() - 31 * 60 * 1000); + fs.utimesSync(lockFile, oldTime, oldTime); + + const s = createSentinel(); + s.registerBuiltInWatchpoints(); + + const result = await s.evaluateWatchpoint('stale-locks'); + expect(result.status).toBe(WatchpointStatus.DEGRADED); + }); + + test('workspace-structure passes with correct dirs', async () => { + // Create required dirs (use .aios-core for backward compat check) + fs.mkdirSync(path.join(TEST_DIR, '.aios-core'), { recursive: true }); + fs.mkdirSync(path.join(TEST_DIR, 'tests'), { recursive: true }); + fs.mkdirSync(path.join(TEST_DIR, 'bin'), { recursive: true }); + + const s = createSentinel(); + s.registerBuiltInWatchpoints(); + + const result = await s.evaluateWatchpoint('workspace-structure'); + expect(result.status).toBe(WatchpointStatus.HEALTHY); + }); +});