feat(observability): add UX readiness health check endpoint (#639)

Author: dealako

apps/lfx-one/otel.mjs

@@ -105,7 +105,7 @@ if (!otlpEndpoint) {
       new HttpInstrumentation({
         ignoreIncomingRequestHook: (req) => {
           const url = req.url || '';
-          return url === '/health' || url === '/api/health' || url.startsWith('/.well-known');
+          return url === '/livez' || url === '/readyz' || url.startsWith('/.well-known');
         },
         applyCustomAttributesOnSpan: (span, request, response) => {
           const req = 'req' in response ? response.req : undefined;

apps/lfx-one/src/server/middleware/auth.middleware.ts

@@ -17,8 +17,9 @@ const TOKEN_EXPIRY_BUFFER_SECONDS = 300;
  * Ordered by specificity - more specific patterns should come first
  */
 const DEFAULT_ROUTE_CONFIG: RouteAuthConfig[] = [
-  // Health check - completely public
-  { pattern: '/health', type: 'api', auth: 'public' },
+  // Liveness and readiness probes - completely public
+  { pattern: '/livez', type: 'api', auth: 'public' },
+  { pattern: '/readyz', type: 'api', auth: 'public' },
 
   // Public API routes - optional authentication with token benefits
   { pattern: '/public/api', type: 'api', auth: 'optional', tokenRequired: false },

apps/lfx-one/src/server/server.ts

@@ -73,6 +73,30 @@ app.use(
 app.use(express.json({ limit: '15mb' }));
 app.use(express.urlencoded({ extended: true, limit: '15mb' }));
 
+// Liveness and readiness endpoints registered before the static handler,
+// logger, auth, and rate-limit middleware so:
+//   - probes are served directly with no filesystem lookup (no I/O overhead
+//     on frequent Kubernetes probe traffic)
+//   - probe traffic is not request-logged
+//   - endpoints are always reachable unauthenticated
+// auth.middleware.ts lists /livez and /readyz as public.
+app.get('/livez', (_req: Request, res: Response) => {
+  res.send('OK');
+});
+
+// Readiness endpoint for Kubernetes (LFXV2-1640).
+// Signals that this pod can accept HTTP traffic: Express is listening and the
+// Angular SSR engine loaded successfully (constructed at module load above —
+// a load failure crashes the process before reaching this point).
+// Intentionally does NOT probe NATS / Snowflake / microservice-proxy: those
+// clients are lazy-initialized and report not-connected at startup even
+// though many SSR pages render fine without them. Per-feature dependency
+// failures are handled at the route level, not by pulling the whole pod out
+// of the Service endpoints list.
+app.get('/readyz', (_req: Request, res: Response) => {
+  res.status(200).json({ status: 'ready' });
+});
+
 app.get(
   '**',
   express.static(browserDistFolder, {
@@ -81,11 +105,6 @@ app.get(
   })
 );
 
-// Health endpoint before logger middleware so health checks aren't logged.
-app.get('/health', (_req: Request, res: Response) => {
-  res.send('OK');
-});
-
 const httpLogger = pinoHttp({
   logger: serverLogger,
   serializers: {

charts/lfx-v2-ui/templates/deployment.yaml

@@ -53,3 +53,15 @@ spec:
               {{- toYaml $config.valueFrom | nindent 14 }}
             {{- end }}
           {{- end }}
+          {{- with .Values.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}

charts/lfx-v2-ui/values.yaml

@@ -152,6 +152,53 @@ resources:
     cpu: 500m
     memory: 512Mi
 
+# Startup probe — gates liveness and readiness during the initial startup window.
+# Once it succeeds, Kubernetes hands off to livenessProbe and readinessProbe.
+# Budget: 10s initial delay + (10s × 30 failures) = 310s (~5 min) maximum,
+# giving headroom over the observed ~4 min cold-start time.
+# Set to null to disable: startupProbe: null
+startupProbe:
+  httpGet:
+    path: /livez
+    port: http
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  timeoutSeconds: 3
+  failureThreshold: 30
+
+# Liveness probe — restarts the pod if it becomes unresponsive at runtime.
+# startupProbe handles the startup window, so initialDelaySeconds is 0 here.
+# Targets GET /livez (plain text "OK"), registered before logging, auth, and
+# rate-limit middleware so it is always reachable if the process is alive.
+# Set to null to disable: livenessProbe: null
+livenessProbe:
+  httpGet:
+    path: /livez
+    port: http
+  initialDelaySeconds: 0
+  periodSeconds: 15
+  timeoutSeconds: 5
+  failureThreshold: 3
+
+# Readiness probe — controls when the pod receives traffic from the Service.
+# startupProbe handles the startup window, so initialDelaySeconds is 0 here.
+# Targets GET /readyz (JSON {"status":"ready"}) which is registered before
+# pino-http so probes don't pollute request logs. The endpoint confirms that
+# Express is listening and the Angular SSR engine loaded; it intentionally
+# does not probe lazy-initialized clients (NATS, Snowflake) because those
+# report not-connected at startup even when most SSR pages render fine without
+# them. See apps/lfx-one/src/server/server.ts for implementation details.
+# Set to null to disable: readinessProbe: null
+readinessProbe:
+  httpGet:
+    path: /readyz
+    port: http
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 3
+  failureThreshold: 3
+  successThreshold: 1
+
 # Environment variables for the application
 # Uses map/object format for deep merging support
 environment:

docs/architecture/backend/ai-service.md

@@ -321,8 +321,8 @@ serverLogger.error('Failed to generate meeting agenda', { error });
 ### Health Monitoring
 
 ```typescript
-// Health check integration
-app.get('/api/health', (req, res) => {
+// Readiness check integration
+app.get('/readyz', (req, res) => {
   const healthStatus = {
     ai_service: {
       configured: !!process.env['AI_PROXY_URL'] && !!process.env['AI_API_KEY'],

docs/architecture/backend/logging-monitoring.md

@@ -672,22 +672,25 @@ logger.success(req, 'user_login', startTime, {
 
 ## 📊 Health Check Filtering
 
-Health check endpoints are excluded from automatic HTTP logging:
+Liveness and readiness endpoints are excluded from automatic HTTP logging by being registered before the `pino-http` middleware:
 
 ```typescript
-// Health check endpoint (added before logger middleware)
-app.get('/health', (_req: Request, res: Response) => {
+// Liveness and readiness endpoints (registered before logger middleware)
+app.get('/livez', (_req: Request, res: Response) => {
   res.send('OK');
 });
+app.get('/readyz', (_req: Request, res: Response) => {
+  res.status(200).json({ status: 'ready' });
+});
 
-// HTTP logger middleware (added after health endpoint)
+// HTTP logger middleware (added after probe endpoints)
 app.use(httpLogger);
 ```
 
 URLs excluded from logging:
 
-- `/health`
-- `/api/health`
+- `/livez`
+- `/readyz`
 - `/.well-known/*`
 
 ## 🎯 Best Practices

docs/architecture/backend/nats-integration.md

@@ -245,8 +245,8 @@ logger.info(undefined, 'nats_connect', 'Connecting to NATS server on demand', {
 ### Health Check Integration
 
 ```typescript
-// Health endpoint includes NATS status
-app.get('/api/health', (req, res) => {
+// Readiness endpoint includes NATS status
+app.get('/readyz', (req, res) => {
   const healthStatus = {
     nats: {
       connected: natsService.isConnected(),

docs/architecture/backend/observability.md

@@ -40,7 +40,7 @@ Tracing activates only when `OTEL_EXPORTER_OTLP_ENDPOINT` is set — leaving it
 
 ## What gets instrumented
 
-- **HTTP** (`HttpInstrumentation`) — incoming and outgoing HTTP requests, with `/health`, `/api/health`, and `/.well-known/*` excluded so health checks don't flood spans.
+- **HTTP** (`HttpInstrumentation`) — incoming and outgoing HTTP requests, with `/livez`, `/readyz`, and `/.well-known/*` excluded so health checks don't flood spans.
 - **Express** (`ExpressInstrumentation`) — per-middleware spans so you can see which middleware runs for a given request.
 - **Undici** (`UndiciInstrumentation`) — spans for `fetch` / `undici` calls (used by `api-client.service.ts` for microservice calls). Content-type headers are captured.
 - **Propagators** — W3C Trace Context + W3C Baggage (`traceparent`, `tracestate`, `baggage` headers) so spans correlate across the NATS boundary and into downstream microservices.

docs/architecture/backend/snowflake-integration.md

@@ -774,8 +774,8 @@ Track these metrics for operational visibility:
 ### Health Check Integration
 
 ```typescript
-// Add to server health endpoint
-app.get('/health', (req, res) => {
+// Add to server liveness endpoint
+app.get('/livez', (req, res) => {
   const snowflakeStats = snowflakeService.getPoolStats();
   const lockStats = snowflakeService.getLockStats();