fix(prisma): add retry for Aurora Serverless v2 connection errors (#121)

## Issue

close #104
close #105

## Problem

The starter kit has three issues with Prisma + Aurora Serverless v2
(auto-pause enabled with `minCapacity: 0`):

1. **Credential leak**: `console.log(process.env.DATABASE_URL)` in
`prisma.ts` outputs the full connection string including password to
CloudWatch Logs.

2. **No runtime retry**: Aurora drops idle connections after
`idle_session_timeout` (60s) and takes ~15s to resume from auto-pause
([docs](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html)).
Without retry, queries fail with transient errors (P1017, ECONNRESET)
and do not recover.

3. **No migration retry**: `migration-runner.ts` runs `prisma db push`
without retry. During `cdk deploy`, Aurora may still be resuming,
causing P1001 ("Can't reach database server") and failing the entire
deployment.

## Solution

- **Remove `console.log(DATABASE_URL)`** to fix the credential leak.
- **Add a Prisma client extension** (`Prisma.defineExtension` with
`$allModels.$allOperations`) that retries transient connection errors
with exponential backoff. Retryable errors: P2024, P1001, P1017,
idle-session timeout, ECONNRESET. Non-retryable errors (auth failures,
schema errors) are thrown immediately.
- **Add retry to `migration-runner.ts`** for `prisma db push` with
exponential backoff (base 3s, max 5 attempts, ~100s worst case within
Lambda 5min timeout). Only P1001 / connection refused are retried.
- **Optimize connection parameters**: `connection_limit=1` (Lambda
handles one request per instance), `connect_timeout=30` (accommodates
auto-pause resume time).

## Changes

- `webapp/src/lib/prisma.ts` — Remove `console.log`, remove verbose
`log` option, add retry extension via `$extends`
- `webapp/src/jobs/migration-runner.ts` — Extract `runPrismaDbPush` with
retry loop, structured logging
- `cdk/lib/constructs/database.ts` — Change connection options to
`?connection_limit=1&connect_timeout=30`

## Verification

- [ ] `console.log(process.env.DATABASE_URL)` is removed
- [ ] After Aurora auto-pause resume, the first request recovers via
retry
- [ ] Non-retryable errors (e.g. auth failure) are thrown immediately
without retry
- [ ] `cdk deploy` succeeds even when Aurora is resuming from 0 ACU
- [ ] `tsc --noEmit` passes
- [ ] `prettier --check` passes

---------

Co-authored-by: Kazuho Cryer-Shinozuka <malaysia.cryer@gmail.com>

Author: konokenj

cdk/lib/constructs/database.ts

@@ -96,9 +96,10 @@ export class Database extends Construct implements ec2.IConnectable {
 
   public getLambdaEnvironment(databaseName: string) {
     const conn = this.getConnectionInfo();
-    // Aurora Serverless v2 cold start takes up to 15 seconds
-    // https://www.prisma.io/docs/orm/prisma-client/setup-and-configuration/databases-connections/connection-pool
-    const option = '?pool_timeout=20&connect_timeout=20';
+    // connection_limit=1: Each Lambda instance handles one request at a time
+    // connect_timeout=30: Aurora Serverless v2 auto-pause resume takes ~15s (longer after 24h+ pause)
+    // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html
+    const option = '?connection_limit=1&connect_timeout=30';
     return {
       DATABASE_HOST: conn.host,
       DATABASE_NAME: databaseName,

cdk/test/__snapshots__/serverless-fullstack-webapp-starter-kit-without-domain.test.ts.snap

@@ -723,7 +723,7 @@ exports[`Snapshot test 2`] = `
               ],
             },
             "DATABASE_NAME": "main",
-            "DATABASE_OPTION": "?pool_timeout=20&connect_timeout=20",
+            "DATABASE_OPTION": "?connection_limit=1&connect_timeout=30",
             "DATABASE_PASSWORD": {
               "Fn::Join": [
                 "",
@@ -772,7 +772,7 @@ exports[`Snapshot test 2`] = `
                       "Endpoint.Port",
                     ],
                   },
-                  "/main?pool_timeout=20&connect_timeout=20",
+                  "/main?connection_limit=1&connect_timeout=30",
                 ],
               ],
             },
@@ -3575,7 +3575,7 @@ async function handler(event) {
               ],
             },
             "DATABASE_NAME": "main",
-            "DATABASE_OPTION": "?pool_timeout=20&connect_timeout=20",
+            "DATABASE_OPTION": "?connection_limit=1&connect_timeout=30",
             "DATABASE_PASSWORD": {
               "Fn::Join": [
                 "",
@@ -3624,7 +3624,7 @@ async function handler(event) {
                       "Endpoint.Port",
                     ],
                   },
-                  "/main?pool_timeout=20&connect_timeout=20",
+                  "/main?connection_limit=1&connect_timeout=30",
                 ],
               ],
             },
@@ -3952,7 +3952,7 @@ async function handler(event) {
               ],
             },
             "DATABASE_NAME": "main",
-            "DATABASE_OPTION": "?pool_timeout=20&connect_timeout=20",
+            "DATABASE_OPTION": "?connection_limit=1&connect_timeout=30",
             "DATABASE_PASSWORD": {
               "Fn::Join": [
                 "",
@@ -4001,7 +4001,7 @@ async function handler(event) {
                       "Endpoint.Port",
                     ],
                   },
-                  "/main?pool_timeout=20&connect_timeout=20",
+                  "/main?connection_limit=1&connect_timeout=30",
                 ],
               ],
             },

cdk/test/__snapshots__/serverless-fullstack-webapp-starter-kit.test.ts.snap

@@ -744,7 +744,7 @@ exports[`Snapshot test 2`] = `
               ],
             },
             "DATABASE_NAME": "main",
-            "DATABASE_OPTION": "?pool_timeout=20&connect_timeout=20",
+            "DATABASE_OPTION": "?connection_limit=1&connect_timeout=30",
             "DATABASE_PASSWORD": {
               "Fn::Join": [
                 "",
@@ -793,7 +793,7 @@ exports[`Snapshot test 2`] = `
                       "Endpoint.Port",
                     ],
                   },
-                  "/main?pool_timeout=20&connect_timeout=20",
+                  "/main?connection_limit=1&connect_timeout=30",
                 ],
               ],
             },
@@ -3405,7 +3405,7 @@ async function handler(event) {
               ],
             },
             "DATABASE_NAME": "main",
-            "DATABASE_OPTION": "?pool_timeout=20&connect_timeout=20",
+            "DATABASE_OPTION": "?connection_limit=1&connect_timeout=30",
             "DATABASE_PASSWORD": {
               "Fn::Join": [
                 "",
@@ -3454,7 +3454,7 @@ async function handler(event) {
                       "Endpoint.Port",
                     ],
                   },
-                  "/main?pool_timeout=20&connect_timeout=20",
+                  "/main?connection_limit=1&connect_timeout=30",
                 ],
               ],
             },
@@ -3758,7 +3758,7 @@ async function handler(event) {
               ],
             },
             "DATABASE_NAME": "main",
-            "DATABASE_OPTION": "?pool_timeout=20&connect_timeout=20",
+            "DATABASE_OPTION": "?connection_limit=1&connect_timeout=30",
             "DATABASE_PASSWORD": {
               "Fn::Join": [
                 "",
@@ -3807,7 +3807,7 @@ async function handler(event) {
                       "Endpoint.Port",
                     ],
                   },
-                  "/main?pool_timeout=20&connect_timeout=20",
+                  "/main?connection_limit=1&connect_timeout=30",
                 ],
               ],
             },

webapp/src/jobs/migration-runner.ts

@@ -27,26 +27,45 @@ export const handler: Handler = async (event, _) => {
   // Currently we don't have any direct method to invoke prisma migration programmatically.
   // As a workaround, we spawn migration script as a child process and wait for its completion.
   // Please also refer to the following GitHub issue: https://github.com/prisma/prisma/issues/4703
-  try {
-    const exitCode = await new Promise((resolve, _) => {
+  await runPrismaDbPush(options);
+};
+
+// Aurora Serverless v2 may be resuming from auto-pause (0 ACU) during CDK deployment,
+// which takes approximately 15 seconds. Retry transient connection errors with exponential backoff.
+// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html
+async function runPrismaDbPush(options: string[], maxRetries = 5, baseDelay = 3000): Promise<void> {
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    const { exitCode, stdout, stderr } = await new Promise<{
+      exitCode: number;
+      stdout: string;
+      stderr: string;
+    }>((resolve) => {
       execFile(
         path.resolve('./node_modules/prisma/build/index.js'),
         ['db', 'push', '--skip-generate'].concat(options),
         (error, stdout, stderr) => {
-          console.log(stdout);
-          if (error != null) {
-            console.log(`prisma db push exited with error ${error.message}`);
-            resolve(error.code ?? 1);
-          } else {
-            resolve(0);
-          }
+          resolve({
+            exitCode: error ? (typeof error.code === 'number' ? error.code : 1) : 0,
+            stdout,
+            stderr,
+          });
         },
       );
     });
 
-    if (exitCode != 0) throw Error(`db push failed with exit code ${exitCode}`);
-  } catch (e) {
-    console.log(e);
-    throw e;
+    console.log(`prisma db push attempt ${attempt}/${maxRetries}`, { exitCode, stdout, stderr });
+
+    if (exitCode === 0) return;
+
+    const isRetryable =
+      stderr.includes('P1001') || stderr.includes("Can't reach database") || stderr.includes('Connection refused');
+
+    if (!isRetryable || attempt === maxRetries) {
+      throw new Error(`prisma db push failed after ${attempt} attempt(s): ${stderr}`);
+    }
+
+    const delay = baseDelay * Math.pow(2, attempt - 1) + Math.random() * 1000;
+    console.log(`Retrying prisma db push in ${Math.round(delay)}ms...`);
+    await new Promise((r) => setTimeout(r, delay));
   }
-};
+}

webapp/src/lib/prisma.ts

@@ -1,12 +1,70 @@
-import { PrismaClient } from '@prisma/client';
+import { Prisma, PrismaClient } from '@prisma/client';
 
 // https://www.prisma.io/docs/guides/nextjs
 
 const globalForPrisma = global as unknown as {
   prisma: PrismaClient;
 };
 
-console.log(process.env.DATABASE_URL);
-export const prisma = globalForPrisma.prisma || new PrismaClient({ log: ['query', 'info', 'warn', 'error'] });
+// Determine if an error is a transient connection issue that may resolve on retry.
+// Aurora Serverless v2 can drop connections due to idle_session_timeout (60s) or auto-pause,
+// and resume takes approximately 15 seconds.
+// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html
+function isRetryableError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  const code = (error as { code?: string }).code;
+  if (
+    code === 'P2024' || // Connection pool timeout
+    code === 'P1001' || // Can't reach database server
+    code === 'P1017' // Server has closed the connection
+  ) {
+    return true;
+  }
+  const msg = error.message;
+  return (
+    msg.includes('idle-session timeout') ||
+    msg.includes('terminating connection') ||
+    msg.includes('Connection terminated') ||
+    msg.includes('Timed out fetching a new connection from the connection pool') ||
+    msg.includes('ECONNRESET')
+  );
+}
+
+const basePrisma = new PrismaClient();
+
+async function withRetry<T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 500): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const result = await fn();
+      if (attempt > 0) {
+        console.warn(`Prisma query succeeded after ${attempt} retry(s)`);
+      }
+      return result;
+    } catch (error) {
+      lastError = error;
+      if (attempt === maxRetries || !isRetryableError(error)) throw error;
+      // Discard stale connections before retrying
+      await basePrisma.$disconnect();
+      const delay = baseDelay * Math.pow(2, attempt) + Math.random() * 100;
+      console.warn(`Prisma retry attempt ${attempt + 1}/${maxRetries}, waiting ${Math.round(delay)}ms`);
+      await new Promise((r) => setTimeout(r, delay));
+    }
+  }
+  throw lastError;
+}
+
+const retryExtension = Prisma.defineExtension({
+  name: 'retry-on-connection-error',
+  query: {
+    $allModels: {
+      async $allOperations({ args, query }) {
+        return withRetry(() => query(args));
+      },
+    },
+  },
+});
+
+export const prisma = basePrisma.$extends(retryExtension) as unknown as PrismaClient;
 
 if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = prisma;