[Refactor] [Fix] Email Rendering Pipeline Refactor, Error Handling, and Bug Fixes (#1140)

### Context
We noticed some errors pop up on sentry related to email rendering.
These errors seem to have been triggered by the same issue, and could be
categorized as follows:

1. Sanity test mismatch, even when the errors from freestyle and vercel
sandbox were broadly similar. This occurred due to stack traces
differing in different execution environments.
2. Rendering errors from freestyle and vercel sandbox caused by the
theme not being imported/ empty theme component.

Upon investigation, this occurred because hitting save on the email
themes page with an invalid theme (ex: deleting the `export` keyword, or
renaming the `EmailTheme` component) still triggers `bundleAndExecute`
with the invalid themes. This will obviously fail and cause the errors
to be logged, however there is no cause for concern here because the
error is returned and the save is denied because an error is returned.
It's more of a matter of noisy error logs and too strict sanity test
comparisons.

Beyond that, `js-execution` is a little opaque and hard to understand,
and this can mask errors in logic.

We also noticed a new issue: manually throwing an error in the email
theme code editor, and then trying to save was actually successful. This
was because the version of `react-email/components` we were using had
faulty error handling, and fell back to client side rendering, masking
the error. This wasn't caught by our `try-catch` safeguards because it
was a render time issue that was masked. More specifically, this was
what `react-email` was doing: `Switched to client rendering because the
server rendering errored`.

### Summary of Changes

We loosen the sanity test comparison between engine execution results in
case of errors. We then refactor the `js-execution` and
`email-rendering` files to read better, and to only `captureError` when
a service is down, but not for runtime errors in the user submitted
code.

To deal with the other bug, we bumped `react-email/components` to the
latest version. However, doing so exposed a gap between real `freestyle`
and our `freestyle-mock`: with the mock, the errors that were now raised
were treated as uncaught exceptions, crashing the mock server.
Consequently, we switched to using `node` over `bun`.

We also expanded test coverage to account for different error paths. 

Co-authored-by: Konstantin Wohlwend <n2d4xc@gmail.com>

Author: nams1570

apps/backend/src/lib/email-rendering.tsx

@@ -1,7 +1,6 @@
-import { executeJavascript } from '@/lib/js-execution';
+import { executeJavascript, type ExecuteResult } from '@/lib/js-execution';
 import { emptyEmailTheme } from '@stackframe/stack-shared/dist/helpers/emails';
-import { getNodeEnvironment } from '@stackframe/stack-shared/dist/utils/env';
-import { captureError, StackAssertionError } from '@stackframe/stack-shared/dist/utils/errors';
+import { StackAssertionError } from '@stackframe/stack-shared/dist/utils/errors';
 import { bundleJavaScript } from '@stackframe/stack-shared/dist/utils/esbuild';
 import { get, has } from '@stackframe/stack-shared/dist/utils/objects';
 import { Result } from "@stackframe/stack-shared/dist/utils/results";
@@ -50,7 +49,7 @@ export function createTemplateComponentFromHtml(html: string) {
 const nodeModules = {
   "react-dom": "19.1.1",
   "react": "19.1.1",
-  "@react-email/components": "0.1.1",
+  "@react-email/components": "1.0.6",
   "arktype": "2.1.20",
 };
 
@@ -70,12 +69,9 @@ const entryJs = deindent`
 `;
 
 type EmailRenderResult = { html: string, text: string, subject?: string, notificationCategory?: string };
-type ExecuteResult =
-  | { status: "ok", data: unknown }
-  | { status: "error", error: unknown };
 
 async function bundleAndExecute<T>(
-  files: Record<string, string> & { '/entry.js': string }
+  files: Record<string, string> & { '/entry.js': string },
 ): Promise<Result<T, string>> {
   const bundle = await bundleJavaScript(files, {
     keepAsImports: ['arktype', 'react', 'react/jsx-runtime', '@react-email/components'],
@@ -87,40 +83,11 @@ async function bundleAndExecute<T>(
     return Result.error(bundle.error);
   }
 
-  if (["development", "test"].includes(getNodeEnvironment())) {
-    const executeResult = await executeJavascript(bundle.data, { nodeModules, engine: 'freestyle' }) as ExecuteResult;
-    if (executeResult.status === "error") {
-      return Result.error(JSON.stringify(executeResult.error));
-    }
-    return Result.ok(executeResult.data as T);
-  }
-
-  const executeResult = await executeJavascript(bundle.data, { nodeModules }) as ExecuteResult;
+  const executeResult: ExecuteResult = await executeJavascript(bundle.data, { nodeModules });
 
   if (executeResult.status === "error") {
-    const vercelResult = await executeJavascript(bundle.data, { nodeModules, engine: 'vercel-sandbox' }) as ExecuteResult;
-    if (vercelResult.status === "error") {
-      captureError("email-rendering-freestyle-and-vercel-runtime-error", new StackAssertionError(
-        "Email rendering failed with both freestyle and vercel-sandbox engines",
-        {
-          freestyleError: executeResult.error,
-          vercelError: vercelResult.error,
-          innerCode: bundle.data,
-          innerOptions: [
-            { nodeModules, engine: 'freestyle' },
-            { nodeModules, engine: 'vercel-sandbox' },
-          ],
-        },
-      ));
-      return Result.error(JSON.stringify(vercelResult.error));
-    }
-    captureError("email-rendering-freestyle-runtime-error", new StackAssertionError(
-      "Email rendering failed with freestyle but succeeded with vercel-sandbox",
-      { freestyleError: executeResult.error, innerCode: bundle.data, innerOptions: { nodeModules, engine: 'freestyle' } }
-    ));
-    return Result.ok(vercelResult.data as T);
+    return Result.error(JSON.stringify(executeResult.error));
   }
-
   return Result.ok(executeResult.data as T);
 }
 

apps/backend/src/lib/js-execution.tsx

@@ -8,18 +8,21 @@ import { Freestyle as FreestyleClient } from 'freestyle-sandboxes';
 
 export type ExecuteJavascriptOptions = {
   nodeModules?: Record<string, string>,
-  engine?: 'freestyle' | 'vercel-sandbox',
 };
 
+export type ExecuteResult =
+  | { status: "ok", data: unknown }
+  | { status: "error", error: { message: string, stack?: string, cause?: unknown } };
+
 type JsEngine = {
   name: string,
-  execute: (code: string, options: ExecuteJavascriptOptions) => Promise<unknown>,
+  execute: (code: string, options: ExecuteJavascriptOptions) => Promise<ExecuteResult>,
 };
 
 function createFreestyleEngine(): JsEngine {
   return {
     name: 'freestyle',
-    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<unknown> => {
+    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> => {
       const apiKey = getEnvVariable("STACK_FREESTYLE_API_KEY");
       let baseUrl = getEnvVariable("STACK_FREESTYLE_API_ENDPOINT", "") || undefined;
 
@@ -45,15 +48,15 @@ function createFreestyleEngine(): JsEngine {
         throw new StackAssertionError("Freestyle execution returned undefined result", { response, innerCode: code, innerOptions: options });
       }
 
-      return response.result;
+      return response.result as ExecuteResult;
     },
   };
 }
 
 function createVercelSandboxEngine(): JsEngine {
   return {
     name: 'vercel-sandbox',
-    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<unknown> => {
+    execute: async (code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> => {
       const teamId = getEnvVariable("STACK_VERCEL_SANDBOX_TEAM_ID", "");
       const projectId = getEnvVariable("STACK_VERCEL_SANDBOX_PROJECT_ID", "");
       const token = getEnvVariable("STACK_VERCEL_SANDBOX_TOKEN", "");
@@ -122,51 +125,66 @@ const engineMap = new Map<string, JsEngine>([
   ['vercel-sandbox', createVercelSandboxEngine()],
 ]);
 
-const engines: JsEngine[] = Array.from(engineMap.values());
-
 /**
  * Executes the given code with the given options. Returns the result of the code execution
  * if it is JSON-serializable. Has undefined behavior if it is not JSON-serializable or if
  * the code throws an error.
  */
-export async function executeJavascript(code: string, options: ExecuteJavascriptOptions = {}): Promise<unknown> {
+export async function executeJavascript(code: string, options: ExecuteJavascriptOptions = {}): Promise<ExecuteResult> {
   return await traceSpan({
     description: 'js-execution.executeJavascript',
     attributes: {
       'js-execution.code.length': code.length.toString(),
       'js-execution.nodeModules.count': options.nodeModules ? Object.keys(options.nodeModules).length.toString() : '0',
-      'js-execution.engine': options.engine ?? 'auto',
     }
   }, async () => {
-    if (options.engine) {
-      const engine = engineMap.get(options.engine);
-      if (!engine) {
-        throw new StackAssertionError(`Unknown JS execution engine: ${options.engine}`);
+
+    if (getEnvVariable("STACK_VERCEL_SANDBOX_TOKEN","") != "") {
+      if (!getNodeEnvironment().includes("prod")) {
+        throw new StackAssertionError("STACK_VERCEL_SANDBOX_TOKEN is set in non-production environment. We do not use Vercel Sandbox in non-production environments.");
       }
-      return await engine.execute(code, options);
-    }
 
-    const shouldSanityTest = Math.random() < 0.05;
+      const shouldSanityTest = Math.random() < 0.05;
+      if (shouldSanityTest) {
+        runAsynchronouslyAndWaitUntil(runSanityTest(code, options));
+      }
 
-    if (shouldSanityTest) {
-      runAsynchronouslyAndWaitUntil(runSanityTest(code, options));
+      return await runWithFallback(code, options);
+    } else {
+      return await runWithoutFallback(code, options);
     }
-
-    // Normal execution: try engines in order with retry for first engine
-    return await runWithFallback(code, options);
   });
 }
 
+/**
+ * Compare two execution results for sanity test equality.
+ * For error results, we only compare status and message (not stack traces,
+ * which differ between execution environments).
+ */
+function areResultsEqual(a: ExecuteResult, b: ExecuteResult): boolean {
+  if (a.status !== b.status) return false;
+
+  if (a.status === 'ok' && b.status === 'ok') {
+    return JSON.stringify(a.data) === JSON.stringify(b.data);
+  }
+
+  if (a.status === 'error' && b.status === 'error') {
+    return a.error.message === b.error.message;
+  }
+
+  return false;
+}
+
 async function runSanityTest(code: string, options: ExecuteJavascriptOptions) {
   const results: Array<{ engine: string, result: unknown }> = [];
   const failures: Array<{ engine: string, error: unknown }> = [];
 
-  for (const engine of engines) {
+  for (const [name, engine] of engineMap) {
     try {
       const result = await engine.execute(code, options);
-      results.push({ engine: engine.name, result });
+      results.push({ engine: name, result });
     } catch (error) {
-      failures.push({ engine: engine.name, error });
+      failures.push({ engine: name, error });
     }
   }
 
@@ -181,8 +199,8 @@ async function runSanityTest(code: string, options: ExecuteJavascriptOptions) {
     return;
   }
 
-  const referenceResult = results[0].result;
-  const allEqual = results.every(r => JSON.stringify(r.result) === JSON.stringify(referenceResult));
+  const referenceResult = results[0].result as ExecuteResult;
+  const allEqual = results.every(r => areResultsEqual(r.result as ExecuteResult, referenceResult));
   if (!allEqual) {
     captureError("js-execution-sanity-test-mismatch", new StackAssertionError(
       "JS execution sanity test: engines returned different results",
@@ -191,19 +209,15 @@ async function runSanityTest(code: string, options: ExecuteJavascriptOptions) {
   }
 }
 
-async function runWithFallback(code: string, options: ExecuteJavascriptOptions): Promise<unknown> {
-  const errors: Array<{ engine: string, error: unknown }> = [];
+async function runWithFallback(code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> {
+  const freestyleEngine = engineMap.get("freestyle")!;
+  const vercelSandboxEngine = engineMap.get("vercel-sandbox")!;
 
-  for (let i = 0; i < engines.length; i++) {
-    const engine = engines[i];
-    const isFirstEngine = i === 0;
-
-    const maxAttempts = isFirstEngine ? 2 : 1;
-
-    const retryResult = await Result.retry(
+  const maxAttempts = 2;
+  const retryResult = await Result.retry(
       async () => {
         try {
-          const result = await engine.execute(code, options);
+          const result = await freestyleEngine.execute(code, options);
           return Result.ok(result);
         } catch (error) {
           return Result.error(error);
@@ -213,20 +227,33 @@ async function runWithFallback(code: string, options: ExecuteJavascriptOptions):
       { exponentialDelayBase: 500 }
     );
 
-    if (retryResult.status === 'ok') {
-      return retryResult.data;
-    }
-
-    const engineError = retryResult.error;
-    errors.push({ engine: engine.name, error: engineError });
+  if (retryResult.status === 'ok') {
+    return retryResult.data;
+  }
 
-    if (i < engines.length - 1) {
-      captureError(`js-execution-${engine.name}-failed`, new StackAssertionError(
-        `JS execution engine '${engine.name}' failed, falling back to next engine`,
-        { error: engineError, attempts: retryResult.attempts, innerCode: code, innerOptions: options }
+  captureError(`js-execution-freestyle-failed`, new StackAssertionError(
+    `JS execution freestyle engine failed, falling back to vercel sandbox engine`,
+    { error: retryResult.error, innerCode: code, innerOptions: options }
+  ));
+
+  try {
+    const result = await vercelSandboxEngine.execute(code, options);
+    return result;
+  } catch (error){
+      captureError(`js-execution-vercel-sandbox-failed`, new StackAssertionError(
+        `JS execution vercel sandbox engine failed after fallback from freestyle engine`,
+        { error: error, innerCode: code, innerOptions: options }
       ));
-    }
+      throw new StackAssertionError("Email rendering service unavailable", { cause: error, innerCode: code, innerOptions: options });
   }
+}
 
-  throw errors[errors.length - 1].error;
+async function runWithoutFallback(code: string, options: ExecuteJavascriptOptions): Promise<ExecuteResult> {
+  const freestyleEngine = engineMap.get("freestyle")!;
+  try {
+    const result = await freestyleEngine.execute(code, options);
+    return result;
+  } catch (error) {
+    throw new StackAssertionError("Email rendering service unavailable", { cause: error, innerCode: code, innerOptions: options });
+  }
 }