atomic update

Author: ajhollid

server/src/repositories/monitors/IMonitorsRepository.ts

@@ -1,4 +1,4 @@
-import { type MonitorType, type Monitor, type MonitorsSummary } from "@/types/index.js";
+import { type MonitorType, type Monitor, type MonitorsSummary, CheckSnapshot } from "@/types/index.js";
 
 export interface TeamQueryConfig {
 	limit?: number;
@@ -29,6 +29,14 @@ export interface IMonitorsRepository {
 
 	// update
 	updateById(monitorId: string, teamId: string, updates: Partial<Monitor>): Promise<Monitor>;
+	updateStatusWindowAndChecks(
+		monitorId: string,
+		teamId: string,
+		status: boolean,
+		checkSnapshot: CheckSnapshot,
+		windowSize: number,
+		maxRecentChecks: number
+	): Promise<Monitor>;
 	togglePauseById(monitorId: string, teamId: string): Promise<Monitor>;
 	// delete
 	deleteById(monitorId: string, teamId: string): Promise<Monitor>;

server/src/repositories/monitors/MongoMonitorsRepository.ts

@@ -182,6 +182,31 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 		return this.toEntity(updatedMonitor);
 	};
 
+	updateStatusWindowAndChecks = async (
+		monitorId: string,
+		teamId: string,
+		status: boolean,
+		checkSnapshot: CheckSnapshot,
+		windowSize: number,
+		maxRecentChecks: number
+	): Promise<Monitor> => {
+		const updatedMonitor = await MonitorModel.findOneAndUpdate(
+			{ _id: monitorId, teamId },
+			{
+				$push: {
+					statusWindow: { $each: [status], $slice: -windowSize },
+					recentChecks: { $each: [checkSnapshot], $slice: -maxRecentChecks },
+				},
+			},
+			{ returnDocument: "after" }
+		);
+
+		if (!updatedMonitor) {
+			throw new AppError({ message: `Failed to update status and checks for monitor with id ${monitorId}`, status: 500 });
+		}
+		return this.toEntity(updatedMonitor);
+	};
+
 	togglePauseById = async (monitorId: string, teamId: string) => {
 		const monitor = await MonitorModel.findOneAndUpdate(
 			{ _id: monitorId, teamId },

server/src/service/infrastructure/statusService.ts

@@ -217,6 +217,125 @@ export class StatusService implements IStatusService {
 		return { nextStatus, transitioned, breaches, nextCounters };
 	};
 
+	private updateStatusWindowAndRecentChecks = async (monitor: Monitor, check: Check) => {
+		const checkSnapshot = this.toCheckSnapshot(check);
+		return await this.monitorsRepository.updateStatusWindowAndChecks(
+			monitor.id,
+			monitor.teamId,
+			check.status,
+			checkSnapshot,
+			monitor.statusWindowSize,
+			MAX_RECENT_CHECKS
+		);
+	};
+
+	newUpdateMonitorStatus = async (
+		statusResponse: MonitorStatusResponse<
+			| PingStatusPayload
+			| HttpStatusPayload
+			| PageSpeedStatusPayload
+			| HardwareStatusPayload
+			| DockerStatusPayload
+			| PortStatusPayload
+			| GameStatusPayload
+			| GrpcStatusPayload
+			| undefined
+		>,
+		check: Check
+	): Promise<StatusChangeResult> => {
+		try {
+			const { monitorId, teamId, status, code } = statusResponse;
+			const monitor = await this.monitorsRepository.findById(monitorId, teamId);
+
+			// Update running stats
+			await this.tryUpdateRunningStats(monitor, statusResponse);
+			const updatedMonitor = await this.updateStatusWindowAndRecentChecks(monitor, check);
+			const prevStatus = updatedMonitor.status;
+
+			// Return early if not enough data points
+			if (updatedMonitor.statusWindow.length < updatedMonitor.statusWindowSize) {
+				updatedMonitor.status = status === true ? "up" : "down";
+				const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, { status: updatedMonitor.status });
+				return {
+					monitor: updated,
+					statusChanged: false,
+					prevStatus,
+					code,
+					timestamp: Date.now(),
+				};
+			}
+
+			// With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition.
+			let newStatus: MonitorStatus;
+			if (updatedMonitor.status === "initializing") {
+				newStatus = status === true ? "up" : "down";
+			} else {
+				newStatus = updatedMonitor.status;
+			}
+
+			let statusChanged = false;
+
+			// First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches.
+			const reachabilityResult = this.computeReachability(newStatus, updatedMonitor.statusWindow, updatedMonitor.statusWindowThreshold);
+			if (reachabilityResult.transitioned) {
+				newStatus = reachabilityResult.nextStatus;
+				statusChanged = true;
+			}
+
+			// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
+			let thresholdBreaches: HardwareBreaches | undefined;
+			const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
+			if (updatedMonitor.type === "hardware" && hardwarePayload?.data) {
+				const hardware = this.computeHardwareStatus({
+					currentStatus: newStatus,
+					reachabilityDown: newStatus === "down",
+					metrics: hardwarePayload.data,
+					thresholds: {
+						cpu: updatedMonitor.cpuAlertThreshold,
+						memory: updatedMonitor.memoryAlertThreshold,
+						disk: updatedMonitor.diskAlertThreshold,
+						temp: updatedMonitor.tempAlertThreshold,
+					},
+					counters: {
+						cpu: updatedMonitor.cpuAlertCounter,
+						memory: updatedMonitor.memoryAlertCounter,
+						disk: updatedMonitor.diskAlertCounter,
+						temp: updatedMonitor.tempAlertCounter,
+					},
+				});
+
+				updatedMonitor.cpuAlertCounter = hardware.nextCounters.cpu;
+				updatedMonitor.memoryAlertCounter = hardware.nextCounters.memory;
+				updatedMonitor.diskAlertCounter = hardware.nextCounters.disk;
+				updatedMonitor.tempAlertCounter = hardware.nextCounters.temp;
+				thresholdBreaches = hardware.breaches;
+				if (hardware.transitioned) {
+					newStatus = hardware.nextStatus;
+					statusChanged = true;
+				}
+			}
+
+			// Apply the final status
+			updatedMonitor.status = newStatus;
+
+			const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, updatedMonitor);
+
+			return {
+				monitor: updated,
+				statusChanged,
+				prevStatus,
+				code,
+				timestamp: new Date().getTime(),
+				thresholdBreaches,
+			};
+		} catch (error: unknown) {
+			throw new AppError({
+				message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`,
+				service: SERVICE_NAME,
+				method: "updateMonitorStatus",
+			});
+		}
+	};
 	updateMonitorStatus = async (
 		statusResponse: MonitorStatusResponse<
 			| PingStatusPayload