Add Datadog log forwarding with pod identity (#60347)

Author: Ebonsignori

config/kubernetes/default/deployments/webapp.yaml

@@ -16,6 +16,8 @@ spec:
         # See https://thehub.github.com/epd/engineering/dev-practicals/observability/logging/ for more details
         fluentbit.io/parser: logfmt_sloppy
         observability.github.com/splunk_index: docs-internal
+        ad.datadoghq.com/webapp.logs: '[{"source":"nodejs","service":"docs-internal","tags":["env:staging"]}]'
+        ad.datadoghq.com/tolerate-unready: 'true'
     spec:
       dnsPolicy: Default
       containers:
@@ -47,6 +49,15 @@ spec:
             # configuration set in config/moda/configuration/*/env.yaml
             - configMapRef:
                 name: application-config
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           # Zero-downtime deploys
           # https://thehub.github.com/engineering/products-and-services/internal/moda/feature-documentation/pod-lifecycle/#required-prestop-hook
           # https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks

config/kubernetes/production/deployments/webapp.yaml

@@ -18,6 +18,8 @@ spec:
         # See https://thehub.github.com/epd/engineering/dev-practicals/observability/logging/ for more details
         fluentbit.io/parser: logfmt_sloppy
         observability.github.com/splunk_index: docs-internal
+        ad.datadoghq.com/webapp.logs: '[{"source":"nodejs","service":"docs-internal","tags":["env:production"]}]'
+        ad.datadoghq.com/tolerate-unready: 'true'
     spec:
       dnsPolicy: Default
       containers:
@@ -50,6 +52,15 @@ spec:
             # configuration set in config/moda/configuration/*/env.yaml
             - configMapRef:
                 name: application-config
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           # Zero-downtime deploys
           # https://thehub.github.com/engineering/products-and-services/internal/moda/feature-documentation/pod-lifecycle/#required-prestop-hook
           # https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks

src/observability/logger/index.ts

@@ -6,6 +6,7 @@ import {
   useProductionLogging,
 } from '@/observability/logger/lib/log-levels'
 import { toLogfmt } from '@/observability/logger/lib/to-logfmt'
+import { POD_IDENTITY } from '@/observability/logger/lib/pod-identity'
 
 type IncludeContext = { [key: string]: unknown }
 
@@ -131,7 +132,8 @@ export function createLogger(filePath: string) {
     if (useProductionLogging()) {
       // Logfmt logging in production
       const logObject: IncludeContext = {
-        ...loggerContext,
+        ...POD_IDENTITY, // pod_name, pod_namespace, node_hostname (static; {} in local dev)
+        ...loggerContext, // requestUuid, path, method, headers, etc. (per-request)
         timestamp,
         level,
         file: path.relative(process.cwd(), new URL(filePath).pathname),

src/observability/logger/lib/pod-identity.ts

@@ -0,0 +1,6 @@
+// Static pod-identity fields — read once at module load, never change.
+// Only populated when running in Kubernetes (env vars set via Downward API + kube-cluster-metadata).
+export const POD_IDENTITY: Record<string, string> = {}
+if (process.env.POD_NAME) POD_IDENTITY.podName = process.env.POD_NAME
+if (process.env.POD_NAMESPACE) POD_IDENTITY.podNamespace = process.env.POD_NAMESPACE
+if (process.env.KUBE_NODE_HOSTNAME) POD_IDENTITY.nodeHostname = process.env.KUBE_NODE_HOSTNAME

src/observability/logger/middleware/get-automatic-request-logger.ts

@@ -3,6 +3,7 @@ import { getLoggerContext } from '@/observability/logger/lib/logger-context'
 import type { NextFunction, Request, Response } from 'express'
 import { getLogLevelNumber, useProductionLogging } from '@/observability/logger/lib/log-levels'
 import { toLogfmt } from '@/observability/logger/lib/to-logfmt'
+import { POD_IDENTITY } from '@/observability/logger/lib/pod-identity'
 
 /**
  * Check if automatic development logging is enabled.
@@ -45,6 +46,7 @@ export function getAutomaticRequestLogger() {
         const loggerContext = getLoggerContext()
         console.log(
           toLogfmt({
+            ...POD_IDENTITY,
             ...loggerContext,
             status,
             responseTime: `${responseTime} ms`,

src/observability/tests/get-automatic-request-logger.ts

@@ -394,4 +394,69 @@ describe('getAutomaticRequestLogger', () => {
       }
     })
   })
+
+  describe('pod identity fields in production logs', () => {
+    // Helper to build a minimal mock res/req and trigger res.end
+    async function runMiddlewareAndCapture(
+      middleware: ReturnType<
+        typeof import('@/observability/logger/middleware/get-automatic-request-logger').getAutomaticRequestLogger
+      >,
+    ): Promise<string[]> {
+      const logs: string[] = []
+      const savedLog = console.log
+      console.log = vi.fn((msg: string) => logs.push(msg))
+
+      const req = { method: 'GET', url: '/test', originalUrl: '/test' }
+      const originalEnd = vi.fn()
+      const res = {
+        statusCode: 200,
+        getHeader: vi.fn(() => '0'),
+        end: originalEnd,
+      }
+      const next = vi.fn()
+
+      middleware(req as Request, res as unknown as Response, next)
+      ;(res as unknown as { end: () => void }).end()
+      await new Promise((resolve) => setTimeout(resolve, 20))
+
+      console.log = savedLog
+      return logs
+    }
+
+    it('should include podName, podNamespace, nodeHostname in logfmt output when env vars are set', async () => {
+      vi.stubEnv('POD_NAME', 'webapp-abc123')
+      vi.stubEnv('POD_NAMESPACE', 'docs-internal-staging-cedar')
+      vi.stubEnv('KUBE_NODE_HOSTNAME', 'ghe-k8s-node-42')
+      vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
+      vi.stubEnv('NODE_ENV', 'production')
+
+      vi.resetModules()
+      const { getAutomaticRequestLogger: freshGetMiddleware } =
+        await import('@/observability/logger/middleware/get-automatic-request-logger')
+
+      const logs = await runMiddlewareAndCapture(freshGetMiddleware())
+      expect(logs).toHaveLength(1)
+      expect(logs[0]).toContain('podName=webapp-abc123')
+      expect(logs[0]).toContain('podNamespace=docs-internal-staging-cedar')
+      expect(logs[0]).toContain('nodeHostname=ghe-k8s-node-42')
+    })
+
+    it('should not include pod identity fields when env vars are absent', async () => {
+      vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
+      vi.stubEnv('NODE_ENV', 'production')
+      delete process.env.POD_NAME
+      delete process.env.POD_NAMESPACE
+      delete process.env.KUBE_NODE_HOSTNAME
+
+      vi.resetModules()
+      const { getAutomaticRequestLogger: freshGetMiddleware } =
+        await import('@/observability/logger/middleware/get-automatic-request-logger')
+
+      const logs = await runMiddlewareAndCapture(freshGetMiddleware())
+      expect(logs).toHaveLength(1)
+      expect(logs[0]).not.toContain('podName=')
+      expect(logs[0]).not.toContain('podNamespace=')
+      expect(logs[0]).not.toContain('nodeHostname=')
+    })
+  })
 })

src/observability/tests/logger.ts

@@ -389,4 +389,46 @@ describe('createLogger', () => {
       expect(logOutput).toContain('included.key=value')
     })
   })
+
+  describe('pod identity fields in production logs', () => {
+    it('should include podName, podNamespace, nodeHostname in logfmt output when env vars are set', async () => {
+      vi.stubEnv('POD_NAME', 'webapp-abc123')
+      vi.stubEnv('POD_NAMESPACE', 'docs-internal-staging-cedar')
+      vi.stubEnv('KUBE_NODE_HOSTNAME', 'ghe-k8s-node-42')
+      vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
+
+      // Reset modules so pod-identity is re-evaluated with the new env vars
+      vi.resetModules()
+      const { createLogger: freshCreateLogger } = await import('@/observability/logger')
+
+      const logger = freshCreateLogger('file:///path/to/test.js')
+      logger.info('Pod identity test')
+
+      expect(consoleLogs).toHaveLength(1)
+      const logOutput = consoleLogs[0]
+      expect(logOutput).toContain('podName=webapp-abc123')
+      expect(logOutput).toContain('podNamespace=docs-internal-staging-cedar')
+      expect(logOutput).toContain('nodeHostname=ghe-k8s-node-42')
+    })
+
+    it('should not include pod identity fields in logfmt output when env vars are absent', async () => {
+      vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
+      // Ensure pod env vars are absent
+      delete process.env.POD_NAME
+      delete process.env.POD_NAMESPACE
+      delete process.env.KUBE_NODE_HOSTNAME
+
+      vi.resetModules()
+      const { createLogger: freshCreateLogger } = await import('@/observability/logger')
+
+      const logger = freshCreateLogger('file:///path/to/test.js')
+      logger.info('No pod identity test')
+
+      expect(consoleLogs).toHaveLength(1)
+      const logOutput = consoleLogs[0]
+      expect(logOutput).not.toContain('podName=')
+      expect(logOutput).not.toContain('podNamespace=')
+      expect(logOutput).not.toContain('nodeHostname=')
+    })
+  })
 })