fix(db): prevent connection pool exhaustion causing 500 errors

## Problem
After ~3 days of uptime, rest-api containers became unhealthy due to
database connection timeouts, causing 'Network error while fetching
document' (500) errors for users.

## Root Cause
- Default pool size was 20 connections per container
- 5 backend containers × 20 = 100 potential DB connections
- DigitalOcean Managed Postgres limit: 25-50 connections
- Once limit hit → new connections timeout → health checks fail (503)
- Docker marks containers unhealthy → Traefik stops routing

## Solution
1. Reduced default DB_POOL_SIZE from 20 → 5 per container
2. Reduced idle timeout from 60s → 30s to release connections faster
3. Added explicit DB_POOL_SIZE env vars to docker-compose.prod.yml

New connection math:
- 2× rest-api × 5 = 10
- 2× hocuspocus-server × 5 = 10
- 1× hocuspocus-worker × 8 = 8
- Total: 28 connections (fits managed DB limits)

## Files Changed
- docker-compose.prod.yml: Added DB_POOL_SIZE and DB_IDLE_TIMEOUT
- packages/hocuspocus.server/src/lib/prisma.ts: Safer defaults

Fixes: Database connection exhaustion after extended uptime

Author: HMarzban

docker-compose.prod.yml

@@ -130,6 +130,9 @@ services:
       DATABASE_URL: ${DATABASE_URL}
       REDIS_HOST: docsplus-redis
       REDIS_PORT: 6379
+      # Limit DB pool to prevent connection exhaustion (5 containers × 5 = 25 max)
+      DB_POOL_SIZE: ${DB_POOL_SIZE:-5}
+      DB_IDLE_TIMEOUT: 30000
       SUPABASE_URL: ${SUPABASE_URL}
       SUPABASE_ANON_KEY: ${SUPABASE_ANON_KEY}
       SUPABASE_SERVICE_ROLE_KEY: ${SUPABASE_SERVICE_ROLE_KEY}
@@ -210,6 +213,9 @@ services:
       DATABASE_URL: ${DATABASE_URL}
       REDIS_HOST: docsplus-redis
       REDIS_PORT: 6379
+      # Limit DB pool to prevent connection exhaustion
+      DB_POOL_SIZE: ${DB_POOL_SIZE:-5}
+      DB_IDLE_TIMEOUT: 30000
       SUPABASE_URL: ${SUPABASE_URL}
       SUPABASE_ANON_KEY: ${SUPABASE_ANON_KEY}
       SUPABASE_SERVICE_ROLE_KEY: ${SUPABASE_SERVICE_ROLE_KEY}
@@ -295,6 +301,9 @@ services:
       DATABASE_URL: ${DATABASE_URL}
       REDIS_HOST: docsplus-redis
       REDIS_PORT: 6379
+      # Worker needs slightly higher pool for background jobs
+      DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
+      DB_IDLE_TIMEOUT: 30000
       SUPABASE_URL: ${SUPABASE_URL}
       SUPABASE_ANON_KEY: ${SUPABASE_ANON_KEY}
       SUPABASE_SERVICE_ROLE_KEY: ${SUPABASE_SERVICE_ROLE_KEY}

packages/hocuspocus.server/src/lib/prisma.ts

@@ -39,12 +39,17 @@ const getDatabaseUrl = (): string => {
 }
 
 // Connection pool configuration
+// IMPORTANT: Keep pool size LOW to prevent connection exhaustion with multiple containers
+// With 5 containers at 5 connections each = 25 total (fits most managed DB limits)
 const poolConfig = {
   connectionString: getDatabaseUrl(),
-  max: parseInt(process.env.DB_POOL_SIZE || process.env.DB_CONNECTION_LIMIT || '20', 10),
-  // Increased idle timeout to prevent connections from closing too quickly
-  // Health checks and low-traffic scenarios shouldn't cause connection churn
-  idleTimeoutMillis: parseInt(process.env.DB_IDLE_TIMEOUT || (process.env.NODE_ENV === 'development' ? '300000' : '60000'), 10), // 5min dev, 1min prod
+  max: parseInt(process.env.DB_POOL_SIZE || process.env.DB_CONNECTION_LIMIT || '5', 10),
+  // Reduced idle timeout to release connections faster (30s prod, 5min dev)
+  // This prevents connection accumulation over time
+  idleTimeoutMillis: parseInt(
+    process.env.DB_IDLE_TIMEOUT || (process.env.NODE_ENV === 'development' ? '300000' : '30000'),
+    10
+  ),
   connectionTimeoutMillis: parseInt(process.env.DB_CONNECT_TIMEOUT || '10000', 10),
   // Enable keep-alive to prevent connection drops
   keepAlive: true,
@@ -56,13 +61,11 @@ const poolConfig = {
   // SSL config:
   // - Production: Enable SSL for managed DBs (DigitalOcean, etc.) - rejectUnauthorized: false for self-signed certs
   // - Development: undefined = no SSL override, uses connectionString defaults (local Docker Postgres typically has no SSL)
-  ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : undefined,
+  ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : undefined
 }
 
 // Create PostgreSQL connection pool
-const pool =
-  globalForPrisma.pool ??
-  new Pool(poolConfig)
+const pool = globalForPrisma.pool ?? new Pool(poolConfig)
 
 // Pool event handlers for monitoring
 pool.on('connect', () => {
@@ -82,13 +85,16 @@ pool.on('acquire', () => {
 
   // Production: warn if pool is getting exhausted (80%+ utilization)
   if (process.env.NODE_ENV === 'production' && stats.total >= poolConfig.max * 0.8) {
-    dbLogger.warn({
-      total: stats.total,
-      idle: stats.idle,
-      waiting: stats.waiting,
-      max: poolConfig.max,
-      utilization: `${Math.round((stats.total / poolConfig.max) * 100)}%`
-    }, 'Database pool utilization high - consider increasing DB_POOL_SIZE')
+    dbLogger.warn(
+      {
+        total: stats.total,
+        idle: stats.idle,
+        waiting: stats.waiting,
+        max: poolConfig.max,
+        utilization: `${Math.round((stats.total / poolConfig.max) * 100)}%`
+      },
+      'Database pool utilization high - consider increasing DB_POOL_SIZE'
+    )
   }
 })
 
@@ -99,53 +105,71 @@ pool.on('error', (err: any) => {
   // 57P01: terminating connection due to administrator command
   // This is normal during database restarts/maintenance - don't log as error
   if (errorCode === '57P01') {
-    dbLogger.debug({
-      code: errorCode,
-      message: 'Database connection terminated (likely during maintenance/restart)'
-    }, 'Connection terminated by database - will be automatically replaced')
+    dbLogger.debug(
+      {
+        code: errorCode,
+        message: 'Database connection terminated (likely during maintenance/restart)'
+      },
+      'Connection terminated by database - will be automatically replaced'
+    )
     return
   }
 
   // 57P02: terminating connection due to administrator command (alternative)
   if (errorCode === '57P02') {
-    dbLogger.debug({
-      code: errorCode,
-      message: 'Database connection terminated'
-    }, 'Connection terminated - will be automatically replaced')
+    dbLogger.debug(
+      {
+        code: errorCode,
+        message: 'Database connection terminated'
+      },
+      'Connection terminated - will be automatically replaced'
+    )
     return
   }
 
   // 57P03: cannot connect now (database startup/shutdown)
   if (errorCode === '57P03') {
-    dbLogger.warn({
-      code: errorCode,
-      message: 'Database temporarily unavailable - connection will retry'
-    }, 'Database connection unavailable')
+    dbLogger.warn(
+      {
+        code: errorCode,
+        message: 'Database temporarily unavailable - connection will retry'
+      },
+      'Database connection unavailable'
+    )
     return
   }
 
   // Connection errors that are recoverable
   const recoverableErrors = ['ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND']
-  if (recoverableErrors.some(code => err.code === code || err.message?.includes(code))) {
-    dbLogger.warn({
-      code: err.code,
-      message: err.message
-    }, 'Database connection error - will retry automatically')
+  if (recoverableErrors.some((code) => err.code === code || err.message?.includes(code))) {
+    dbLogger.warn(
+      {
+        code: err.code,
+        message: err.message
+      },
+      'Database connection error - will retry automatically'
+    )
     return
   }
 
   // Log other errors as actual errors
-  dbLogger.error({
-    err,
-    code: errorCode,
-    message: err.message
-  }, 'Unexpected database pool error')
+  dbLogger.error(
+    {
+      err,
+      code: errorCode,
+      message: err.message
+    },
+    'Unexpected database pool error'
+  )
 })
 
 pool.on('remove', () => {
   // Only log in development - connection removal is normal (idle timeout, errors, etc.)
   if (process.env.NODE_ENV === 'development') {
-    dbLogger.debug({ poolSize: pool.totalCount, idle: pool.idleCount }, 'Connection removed from pool')
+    dbLogger.debug(
+      { poolSize: pool.totalCount, idle: pool.idleCount },
+      'Connection removed from pool'
+    )
   }
 })
 
@@ -157,15 +181,14 @@ export const prisma =
   globalForPrisma.prisma ??
   new PrismaClient({
     adapter,
-    log: process.env.NODE_ENV === 'development'
-      ? [
-          { level: 'query', emit: 'event' },
-          { level: 'error', emit: 'event' },
-          { level: 'warn', emit: 'event' }
-        ]
-      : [
-          { level: 'error', emit: 'event' }
-        ]
+    log:
+      process.env.NODE_ENV === 'development'
+        ? [
+            { level: 'query', emit: 'event' },
+            { level: 'error', emit: 'event' },
+            { level: 'warn', emit: 'event' }
+          ]
+        : [{ level: 'error', emit: 'event' }]
   })
 
 // Log Prisma events
@@ -190,11 +213,14 @@ if (process.env.NODE_ENV !== 'production') {
 }
 
 // Log pool configuration on startup
-dbLogger.info({
-  maxConnections: poolConfig.max,
-  idleTimeout: poolConfig.idleTimeoutMillis,
-  connectionTimeout: poolConfig.connectionTimeoutMillis
-}, 'Database connection pool initialized')
+dbLogger.info(
+  {
+    maxConnections: poolConfig.max,
+    idleTimeout: poolConfig.idleTimeoutMillis,
+    connectionTimeout: poolConfig.connectionTimeoutMillis
+  },
+  'Database connection pool initialized'
+)
 
 /**
  * Get current pool statistics
@@ -220,12 +246,15 @@ export const checkDatabaseHealth = async (): Promise<boolean> => {
     // Log pool stats in production if pool is getting exhausted
     const stats = getPoolStats()
     if (process.env.NODE_ENV === 'production' && stats.waiting > 0) {
-      dbLogger.warn({
-        waiting: stats.waiting,
-        total: stats.total,
-        idle: stats.idle,
-        max: poolConfig.max
-      }, 'Database pool has waiting connections - potential bottleneck')
+      dbLogger.warn(
+        {
+          waiting: stats.waiting,
+          total: stats.total,
+          idle: stats.idle,
+          max: poolConfig.max
+        },
+        'Database pool has waiting connections - potential bottleneck'
+      )
     }
 
     return true