fix(prisma): add retry for Aurora Serverless v2 connection errors (#104, #105)

Why: Aurora Serverless v2 with auto-pause (0 ACU) drops connections on
idle_session_timeout and takes ~15s to resume. Without retry, both
runtime queries and CDK deployment migrations fail on transient errors.
Also, DATABASE_URL (including password) was logged to CloudWatch.

What:
- Remove console.log(DATABASE_URL) that leaked credentials to CloudWatch
- Add Prisma client extension with retry on transient connection errors
  (P2024, P1001, P1017, idle-session timeout, ECONNRESET)
- Add exponential backoff retry to migration-runner for prisma db push
- Optimize connection params: connection_limit=1, connect_timeout=30

Author: konokenj

cdk/lib/constructs/database.ts

@@ -90,9 +90,10 @@ export class Database extends Construct implements ec2.IConnectable {
 
   public getLambdaEnvironment(databaseName: string) {
     const conn = this.getConnectionInfo();
-    // Aurora Serverless v2 cold start takes up to 15 seconds
-    // https://www.prisma.io/docs/orm/prisma-client/setup-and-configuration/databases-connections/connection-pool
-    const option = '?pool_timeout=20&connect_timeout=20';
+    // connection_limit=1: Each Lambda instance handles one request at a time
+    // connect_timeout=30: Aurora Serverless v2 auto-pause resume takes ~15s (longer after 24h+ pause)
+    // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html
+    const option = '?connection_limit=1&connect_timeout=30';
     return {
       DATABASE_HOST: conn.host,
       DATABASE_NAME: databaseName,

webapp/src/jobs/migration-runner.ts

@@ -27,26 +27,45 @@ export const handler: Handler = async (event, _) => {
   // Currently we don't have any direct method to invoke prisma migration programmatically.
   // As a workaround, we spawn migration script as a child process and wait for its completion.
   // Please also refer to the following GitHub issue: https://github.com/prisma/prisma/issues/4703
-  try {
-    const exitCode = await new Promise((resolve, _) => {
+  await runPrismaDbPush(options);
+};
+
+// Aurora Serverless v2 may be resuming from auto-pause (0 ACU) during CDK deployment,
+// which takes approximately 15 seconds. Retry transient connection errors with exponential backoff.
+// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html
+async function runPrismaDbPush(options: string[], maxRetries = 5, baseDelay = 3000): Promise<void> {
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    const { exitCode, stdout, stderr } = await new Promise<{
+      exitCode: number;
+      stdout: string;
+      stderr: string;
+    }>((resolve) => {
       execFile(
         path.resolve('./node_modules/prisma/build/index.js'),
         ['db', 'push', '--skip-generate'].concat(options),
         (error, stdout, stderr) => {
-          console.log(stdout);
-          if (error != null) {
-            console.log(`prisma db push exited with error ${error.message}`);
-            resolve(error.code ?? 1);
-          } else {
-            resolve(0);
-          }
+          resolve({
+            exitCode: error ? (typeof error.code === 'number' ? error.code : 1) : 0,
+            stdout,
+            stderr,
+          });
         },
       );
     });
 
-    if (exitCode != 0) throw Error(`db push failed with exit code ${exitCode}`);
-  } catch (e) {
-    console.log(e);
-    throw e;
+    console.log(`prisma db push attempt ${attempt}/${maxRetries}`, { exitCode, stdout, stderr });
+
+    if (exitCode === 0) return;
+
+    const isRetryable =
+      stderr.includes('P1001') || stderr.includes("Can't reach database") || stderr.includes('Connection refused');
+
+    if (!isRetryable || attempt === maxRetries) {
+      throw new Error(`prisma db push failed after ${attempt} attempt(s): ${stderr}`);
+    }
+
+    const delay = baseDelay * Math.pow(2, attempt - 1) + Math.random() * 1000;
+    console.log(`Retrying prisma db push in ${Math.round(delay)}ms...`);
+    await new Promise((r) => setTimeout(r, delay));
   }
-};
+}

webapp/src/lib/prisma.ts

@@ -1,12 +1,65 @@
-import { PrismaClient } from '@prisma/client';
+import { Prisma, PrismaClient } from '@prisma/client';
 
 // https://www.prisma.io/docs/guides/nextjs
 
 const globalForPrisma = global as unknown as {
   prisma: PrismaClient;
 };
 
-console.log(process.env.DATABASE_URL);
-export const prisma = globalForPrisma.prisma || new PrismaClient({ log: ['query', 'info', 'warn', 'error'] });
+// Determine if an error is a transient connection issue that may resolve on retry.
+// Aurora Serverless v2 can drop connections due to idle_session_timeout (60s) or auto-pause,
+// and resume takes approximately 15 seconds.
+// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2-auto-pause.html
+function isRetryableError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  const code = (error as { code?: string }).code;
+  if (
+    code === 'P2024' || // Connection pool timeout
+    code === 'P1001' || // Can't reach database server
+    code === 'P1017' // Server has closed the connection
+  ) {
+    return true;
+  }
+  const msg = error.message;
+  return (
+    msg.includes('idle-session timeout') ||
+    msg.includes('terminating connection') ||
+    msg.includes('Connection terminated') ||
+    msg.includes('ECONNRESET')
+  );
+}
+
+const basePrisma = new PrismaClient();
+
+async function withRetry<T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 500): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt === maxRetries || !isRetryableError(error)) throw error;
+      // Discard stale connections before retrying
+      await basePrisma.$disconnect();
+      const delay = baseDelay * Math.pow(2, attempt) + Math.random() * 100;
+      console.warn(`Prisma retry attempt ${attempt + 1}/${maxRetries}, waiting ${Math.round(delay)}ms`);
+      await new Promise((r) => setTimeout(r, delay));
+    }
+  }
+  throw lastError;
+}
+
+const retryExtension = Prisma.defineExtension({
+  name: 'retry-on-connection-error',
+  query: {
+    $allModels: {
+      async $allOperations({ args, query }) {
+        return withRetry(() => query(args));
+      },
+    },
+  },
+});
+
+export const prisma = basePrisma.$extends(retryExtension) as unknown as PrismaClient;
 
 if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = prisma;