Add Node.js runtime metrics (heap, GC, event loop) to Datadog (#60367)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: heiskr

src/frame/start-server.ts

@@ -5,6 +5,7 @@ import dotenv from 'dotenv'
 
 import { checkNodeVersion } from './lib/check-node-version'
 import '../observability/lib/handle-exceptions'
+import { startRuntimeMetrics } from '@/observability/lib/runtime-metrics'
 import createApp from './lib/app'
 import warmServer from './lib/warm-server'
 import { createLogger } from '@/observability/logger'
@@ -55,6 +56,8 @@ async function startServer() {
   // Workaround for https://github.com/expressjs/express/issues/1101
   const server = http.createServer(app)
 
+  startRuntimeMetrics()
+
   process.once('SIGTERM', () => {
     logger.info('Received SIGTERM, beginning graceful shutdown', { pid: process.pid, port })
     server.close(() => {

src/observability/lib/runtime-metrics.ts

@@ -0,0 +1,76 @@
+/**
+ * Periodically emits Node.js runtime metrics to Datadog via StatsD.
+ *
+ * Covers three categories that are otherwise invisible:
+ *  1. V8 heap — used vs limit, so we can spot memory pressure before OOMs.
+ *  2. GC — pause duration, so we can correlate latency spikes with GC.
+ *  3. Event-loop delay — p50/p99, so we can see when the loop is blocked.
+ *
+ * Only activates when StatsD is sending real metrics (MODA_PROD_SERVICE_ENV).
+ */
+import v8 from 'node:v8'
+import { monitorEventLoopDelay, PerformanceObserver } from 'node:perf_hooks'
+
+import statsd from './statsd'
+
+export const INTERVAL_MS = 10_000
+
+let started = false
+
+function isMetricsEnabled(): boolean {
+  return process.env.MODA_PROD_SERVICE_ENV === 'true' && process.env.NODE_ENV !== 'test'
+}
+
+/**
+ * Call once at server start. Safe to call multiple times (no-op after first).
+ * Only starts collection when StatsD is sending real metrics.
+ */
+export function startRuntimeMetrics(): void {
+  if (started) return
+  started = true
+
+  if (!isMetricsEnabled()) return
+
+  // --- V8 heap stats (sampled on an interval) ---
+  setInterval(() => {
+    const heap = v8.getHeapStatistics()
+    statsd.gauge('node.heap.used', heap.used_heap_size)
+    statsd.gauge('node.heap.total', heap.total_heap_size)
+    statsd.gauge('node.heap.limit', heap.heap_size_limit)
+    statsd.gauge('node.heap.external', heap.external_memory)
+    // Percentage of heap limit currently in use
+    const pct = heap.heap_size_limit > 0 ? (heap.used_heap_size / heap.heap_size_limit) * 100 : 0
+    statsd.gauge('node.heap.used_pct', pct)
+  }, INTERVAL_MS).unref()
+
+  // --- GC pause durations ---
+  const gcObserver = new PerformanceObserver((list) => {
+    for (const entry of list.getEntries()) {
+      const kind = (entry as unknown as { detail?: { kind?: number } }).detail?.kind
+      // kind: 1 = Scavenge (minor), 2 = Mark-Sweep-Compact (major),
+      // 4 = Incremental marking, 8 = Process weak callbacks, 15 = All
+      const tag = kind === 1 ? 'minor' : kind === 2 ? 'major' : 'other'
+      statsd.histogram('node.gc.pause', entry.duration, [`gc_type:${tag}`])
+    }
+  })
+  gcObserver.observe({ entryTypes: ['gc'] })
+
+  // --- Event-loop delay (histogram sampled every 20 ms) ---
+  const eld = monitorEventLoopDelay({ resolution: 20 })
+  eld.enable()
+
+  setInterval(() => {
+    // Values are in nanoseconds; convert to milliseconds for readability.
+    statsd.gauge('node.eventloop.delay.p50', eld.percentile(50) / 1e6)
+    statsd.gauge('node.eventloop.delay.p99', eld.percentile(99) / 1e6)
+    statsd.gauge('node.eventloop.delay.max', eld.max / 1e6)
+    eld.reset()
+  }, INTERVAL_MS).unref()
+}
+
+/**
+ * Reset the started flag. Only for use in tests.
+ */
+export function _resetForTesting(): void {
+  started = false
+}

src/observability/tests/runtime-metrics.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+
+import statsd from '@/observability/lib/statsd'
+import {
+  startRuntimeMetrics,
+  _resetForTesting,
+  INTERVAL_MS,
+} from '@/observability/lib/runtime-metrics'
+
+vi.mock('@/observability/lib/statsd', () => ({
+  default: {
+    gauge: vi.fn(),
+    histogram: vi.fn(),
+  },
+}))
+
+describe('startRuntimeMetrics', () => {
+  beforeEach(() => {
+    _resetForTesting()
+    vi.useFakeTimers()
+    vi.clearAllMocks()
+  })
+
+  afterEach(() => {
+    vi.unstubAllEnvs()
+    vi.useRealTimers()
+  })
+
+  it('is a no-op in test / non-prod environments', () => {
+    vi.stubEnv('MODA_PROD_SERVICE_ENV', 'false')
+    startRuntimeMetrics()
+    vi.advanceTimersByTime(INTERVAL_MS + 1)
+    expect(statsd.gauge).not.toHaveBeenCalled()
+  })
+
+  it('is idempotent — second call does nothing extra', () => {
+    vi.stubEnv('MODA_PROD_SERVICE_ENV', 'true')
+    vi.stubEnv('NODE_ENV', 'production')
+    startRuntimeMetrics()
+    // Second call without reset — should be a no-op
+    startRuntimeMetrics()
+    vi.advanceTimersByTime(INTERVAL_MS + 1)
+    const callCount = (statsd.gauge as ReturnType<typeof vi.fn>).mock.calls.length
+
+    vi.clearAllMocks()
+    vi.advanceTimersByTime(INTERVAL_MS)
+    const secondTickCount = (statsd.gauge as ReturnType<typeof vi.fn>).mock.calls.length
+    // Same number of calls each tick — no duplicate timers registered
+    expect(secondTickCount).toBe(callCount)
+  })
+
+  it('emits heap gauges when enabled', () => {
+    vi.stubEnv('MODA_PROD_SERVICE_ENV', 'true')
+    vi.stubEnv('NODE_ENV', 'production')
+    startRuntimeMetrics()
+    vi.advanceTimersByTime(INTERVAL_MS + 1)
+
+    const gaugeNames = (statsd.gauge as ReturnType<typeof vi.fn>).mock.calls.map(
+      (c: unknown[]) => c[0],
+    )
+    expect(gaugeNames).toContain('node.heap.used')
+    expect(gaugeNames).toContain('node.heap.total')
+    expect(gaugeNames).toContain('node.heap.limit')
+    expect(gaugeNames).toContain('node.heap.external')
+    expect(gaugeNames).toContain('node.heap.used_pct')
+  })
+
+  it('emits event-loop delay gauges when enabled', () => {
+    vi.stubEnv('MODA_PROD_SERVICE_ENV', 'true')
+    vi.stubEnv('NODE_ENV', 'production')
+    startRuntimeMetrics()
+    vi.advanceTimersByTime(INTERVAL_MS + 1)
+
+    const gaugeNames = (statsd.gauge as ReturnType<typeof vi.fn>).mock.calls.map(
+      (c: unknown[]) => c[0],
+    )
+    expect(gaugeNames).toContain('node.eventloop.delay.p50')
+    expect(gaugeNames).toContain('node.eventloop.delay.p99')
+    expect(gaugeNames).toContain('node.eventloop.delay.max')
+  })
+
+  it('emits heap values that are positive numbers', () => {
+    vi.stubEnv('MODA_PROD_SERVICE_ENV', 'true')
+    vi.stubEnv('NODE_ENV', 'production')
+    startRuntimeMetrics()
+    vi.advanceTimersByTime(INTERVAL_MS + 1)
+
+    const heapUsedCall = (statsd.gauge as ReturnType<typeof vi.fn>).mock.calls.find(
+      (c: unknown[]) => c[0] === 'node.heap.used',
+    )
+    expect(heapUsedCall).toBeDefined()
+    expect(heapUsedCall![1]).toBeGreaterThan(0)
+
+    const heapPctCall = (statsd.gauge as ReturnType<typeof vi.fn>).mock.calls.find(
+      (c: unknown[]) => c[0] === 'node.heap.used_pct',
+    )
+    expect(heapPctCall).toBeDefined()
+    expect(heapPctCall![1]).toBeGreaterThan(0)
+    expect(heapPctCall![1]).toBeLessThan(100)
+  })
+})