fix: allow retrying eval on serve testing timeouts

Author: devversion

runner/orchestration/serve-testing-worker.ts

@@ -31,84 +31,74 @@ export async function serveAndTestApp(
 
   progress.log(rootPromptDef, 'serve-testing', `Validating the running app`);
 
-  try {
-    const result = await env.executor.serveWebApplication(
-      evalID,
-      appDirectoryPath,
-      rootPromptDef,
-      progress,
-      abortSignal,
-      async serveUrl => {
-        progress.log(
-          rootPromptDef,
-          'serve-testing',
-          `Validating the running app (URL: ${serveUrl})`,
-        );
-        const serveParams: ServeTestingWorkerMessage = {
-          serveUrl,
-          appName: rootPromptDef.name,
-          enableAutoCsp: !!config.enableAutoCsp,
-          includeAxeTesting: config.skipAxeTesting === false,
-          takeScreenshots: config.skipScreenshots === false,
-          includeLighthouseData: config.skipLighthouse !== true,
-          userJourneyAgentTaskInput,
-        };
+  const result = await env.executor.serveWebApplication(
+    evalID,
+    appDirectoryPath,
+    rootPromptDef,
+    progress,
+    abortSignal,
+    async serveUrl => {
+      progress.log(rootPromptDef, 'serve-testing', `Validating the running app (URL: ${serveUrl})`);
+      const serveParams: ServeTestingWorkerMessage = {
+        serveUrl,
+        appName: rootPromptDef.name,
+        enableAutoCsp: !!config.enableAutoCsp,
+        includeAxeTesting: config.skipAxeTesting === false,
+        takeScreenshots: config.skipScreenshots === false,
+        includeLighthouseData: config.skipLighthouse !== true,
+        userJourneyAgentTaskInput,
+      };
 
-        return await workerConcurrencyQueue.add(
-          () =>
-            new Promise<ServeTestingResult>((resolve, reject) => {
-              const child: ChildProcess = fork(
-                path.resolve(import.meta.dirname, '../workers/serve-testing/worker.js'),
-                {signal: abortSignal},
-              );
-              child.send(serveParams);
+      return await workerConcurrencyQueue.add(
+        () =>
+          new Promise<ServeTestingResult>((resolve, reject) => {
+            const child: ChildProcess = fork(
+              path.resolve(import.meta.dirname, '../workers/serve-testing/worker.js'),
+              {signal: abortSignal},
+            );
+            child.send(serveParams);
 
-              child.on('message', async (result: ServeTestingWorkerResponseMessage) => {
-                if (result.type === 'result') {
-                  try {
-                    await killChildProcessWithSigterm(child);
-                  } catch (e) {
-                    progress.debugLog(`Error while killing serve testing worker: ${e}`);
-                  }
-                  resolve(result.payload);
-                } else {
-                  progress.log(
-                    rootPromptDef,
-                    result.payload.state,
-                    result.payload.message,
-                    result.payload.details,
-                  );
-                }
-              });
-              child.on('error', async err => {
+            child.on('message', async (result: ServeTestingWorkerResponseMessage) => {
+              if (result.type === 'result') {
                 try {
                   await killChildProcessWithSigterm(child);
                 } catch (e) {
                   progress.debugLog(`Error while killing serve testing worker: ${e}`);
                 }
-                reject(err);
-              });
-            }),
-        );
-      },
-    );
-
-    // An executor might define `serveWebApplication` but conditionally decide
-    // that no web application can be started/served.
-    if (result === null) {
-      return null;
-    }
+                resolve(result.payload);
+              } else {
+                progress.log(
+                  rootPromptDef,
+                  result.payload.state,
+                  result.payload.message,
+                  result.payload.details,
+                );
+              }
+            });
+            child.on('error', async err => {
+              try {
+                await killChildProcessWithSigterm(child);
+              } catch (e) {
+                progress.debugLog(`Error while killing serve testing worker: ${e}`);
+              }
+              reject(err);
+            });
+          }),
+      );
+    },
+  );
 
-    if (result.errorMessage === undefined) {
-      progress.log(rootPromptDef, 'success', 'Validation of running app is successful');
-    } else {
-      progress.log(rootPromptDef, 'error', 'Validation of running app failed', result.errorMessage);
-    }
+  // An executor might define `serveWebApplication` but conditionally decide
+  // that no web application can be started/served.
+  if (result === null) {
+    return null;
+  }
 
-    return result;
-  } catch (e) {
-    progress.log(rootPromptDef, 'error', 'Error while trying to validate running app', `${e}`);
+  if (result.errorMessage === undefined) {
+    progress.log(rootPromptDef, 'success', 'Validation of running app is successful');
+  } else {
+    progress.log(rootPromptDef, 'error', 'Validation of running app failed', result.errorMessage);
   }
 
-  return null;
+  return result;
 }

runner/workers/serve-testing/serve-app.ts

@@ -24,7 +24,9 @@ export async function serveApp<T>(
     );
 
     const actualPort = await new Promise<number>((resolvePort, rejectPort) => {
-      const serveStartTimeout = 45000; // 45s for serve to start
+      // Timeout for server to start. CPU queueing might cause this to take longer.
+      // It's a upper safety boundary. The overall eval timeout is the main control.
+      const serveStartTimeout = 1000 * 60 * 5;
       const timeoutId = setTimeout(() => {
         rejectPort(
           new Error(