refactor: ensure errors with dynamic logging are not swallowed

Ensures errors printed with dynamic logging are not swallowed. This is
easily confusing as the report summary is currently just printed with
e.g. zero completed app generations.

Author: devversion

package.json

@@ -5,6 +5,7 @@
     "build-runner": "tsc",
     "release-build": "tsx release-build.ts",
     "web-codegen-scorer": "tsx ./runner/bin/cli.ts",
+    "wcs": "pnpm -s web-codegen-scorer",
     "eval": "pnpm web-codegen-scorer eval",
     "init": "pnpm web-codegen-scorer init",
     "report": "cd report-app && CODEGEN_REPORTS_DIR=../.web-codegen-scorer/reports pnpm start",

runner/eval-cli.ts

@@ -119,7 +119,12 @@ function builder(argv: Argv): Argv<Options> {
     })
     .option('logging', {
       type: 'string',
-      default: 'dynamic' as const,
+      default:
+        process.env['CI'] === '1'
+          ? ('text-only' as const)
+          : ('dynamic' as const),
+      defaultDescription: '`dynamic` (or `text-only` when `CI=1`)',
+      requiresArg: true,
       choices: ['text-only', 'dynamic'] as const,
       description: 'Type of logging to use during the evaluation process',
     })

runner/orchestration/generate.ts

@@ -175,9 +175,9 @@ export async function generateCodeAndAssess(options: {
               stack: e instanceof Error ? e.stack : undefined,
             });
 
-            let details = `  Error: ${e}`;
+            let details = `Error: ${e}`;
             if (e instanceof Error && e.stack) {
-              details += e.stack;
+              details += `\nStack: ${e.stack}`;
             }
 
             progress.log(

runner/progress/dynamic-progress-logger.ts

@@ -6,6 +6,7 @@ import {
   ProgressType,
   progressTypeToIcon,
 } from './progress-logger.js';
+import { redX } from '../reporting/format.js';
 
 const PREFIX_WIDTH = 20;
 
@@ -17,6 +18,11 @@ export class DynamicProgressLogger implements ProgressLogger {
   private spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
   private currentSpinnerFrame = 0;
   private spinnerInterval: ReturnType<typeof setInterval> | undefined;
+  private errors: {
+    prompt: RootPromptDefinition;
+    message: string;
+    details?: string;
+  }[] = [];
 
   initialize(total: number): void {
     this.finalize();
@@ -72,9 +78,22 @@ export class DynamicProgressLogger implements ProgressLogger {
     this.wrapper?.stop();
     this.pendingBars.clear();
     this.wrapper = this.totalBar = this.spinnerInterval = undefined;
+
+    for (const error of this.errors) {
+      let message = `${redX()} [${error.prompt.name}] ${error.message}`;
+      if (error.details) {
+        message += `\n  ${error.details}`;
+      }
+      console.error(message);
+    }
   }
 
-  log(prompt: RootPromptDefinition, type: ProgressType, message: string): void {
+  log(
+    prompt: RootPromptDefinition,
+    type: ProgressType,
+    message: string,
+    details?: string
+  ): void {
     if (!this.wrapper || !this.totalBar) {
       return;
     }
@@ -92,6 +111,11 @@ export class DynamicProgressLogger implements ProgressLogger {
       return;
     }
 
+    // Capture errors for static printing once the dynamic progress is hidden.
+    if (type === 'error') {
+      this.errors.push({ prompt, message, details });
+    }
+
     // Pad/trim the name so they're all the same length.
     const name = this.trimString(
       prompt.name.padEnd(PREFIX_WIDTH, ' '),