build-serve-loop.ts

import PQueue from 'p-queue';
import { LlmGenerateFilesResponse } from '../codegen/llm-runner.js';
import { BuildResultStatus } from '../workers/builder/builder-types.js';
import { Environment } from '../configuration/environment.js';
import {
  AttemptDetails,
  LlmContextFile,
  RootPromptDefinition,
} from '../shared-interfaces.js';
import { DEFAULT_MAX_REPAIR_ATTEMPTS } from '../configuration/constants.js';
import { ProgressLogger } from '../progress/progress-logger.js';
import { runBuild } from './build-worker.js';
import { repairAndBuild } from './build-repair.js';
import { EvalID, Gateway } from './gateway.js';
import { serveAndTestApp } from './serve-testing-worker.js';
import { BrowserAgentTaskInput } from '../testing/browser-agent/models.js';

/**
 * Attempts to build the code that an LLM generated. If the build fails, attempts
 * to fix the breakage and build again.
 *
 * @param evalID ID of the eval being attempted for build.
 * @param gateway Gateway.
 * @param model Model to be used for repair generation requests.
 * @param env Environment that is currently being run.
 * @param rootPromptDef Definition of the root prompt.
 * @param directory Directory on disk to which to write.
 * @param contextFiles Files that should be passed as context to the LLM.
 * @param initialOutputFiles Initial files generated by the LLM.
 * @param usage Usage data from the initial LLM run.
 * @param attemptDetails Array tracking information about the current build attempt.
 * @param skipScreenshots Whether to skip taking screenshots of the app.
 * @param skipAxeTesting Whether or not to skip Axe testing of the app.
 * @param abortSignal Signal to fire when the build should be aborted.
 * @param workerConcurrencyQueue Concurrency queue for controlling parallelism of worker invocations (as they are more expensive than LLM calls).
 */
export async function attemptBuild(
  evalID: EvalID,
  gateway: Gateway<Environment>,
  model: string,
  env: Environment,
  rootPromptDef: RootPromptDefinition,
  directory: string,
  contextFiles: LlmContextFile[],
  initialResponse: LlmGenerateFilesResponse,
  attemptDetails: AttemptDetails[],
  abortSignal: AbortSignal,
  workerConcurrencyQueue: PQueue,
  progress: ProgressLogger,
  skipScreenshots: boolean,
  skipAxeTesting: boolean,
  enableAutoCsp: boolean,
  userJourneyAgentTaskInput?: BrowserAgentTaskInput
) {
  // Clone the original files, because we're going to mutate them between repair
  // attempts and we don't want the different runs to influence each other.
  const finalOutputFiles = initialResponse.files.map((file) => ({
    ...file,
  }));
  const initialBuildResult = await runBuild(
    evalID,
    gateway,
    directory,
    env,
    rootPromptDef,
    abortSignal,
    workerConcurrencyQueue,
    progress
  );
  let repairAttempts = 0;
  const maxRepairAttempts = gateway.shouldRetryFailedBuilds(evalID)
    ? DEFAULT_MAX_REPAIR_ATTEMPTS
    : 0;

  const initialAttempt = {
    outputFiles: initialResponse.files,
    usage: {
      ...{ inputTokens: 0, outputTokens: 0, totalTokens: 0 },
      ...initialResponse.usage,
    },
    reasoning: initialResponse.reasoning,
    buildResult: initialBuildResult,
    serveTestingResult: null,
    attempt: 0,
  };
  attemptDetails.push(initialAttempt);

  let lastAttempt: AttemptDetails = initialAttempt;
  while (
    lastAttempt.buildResult.status !== BuildResultStatus.SUCCESS &&
    repairAttempts < maxRepairAttempts
  ) {
    repairAttempts++;
    progress.log(
      rootPromptDef,
      'build',
      `Trying to repair app build (attempt #${repairAttempts + 1})`
    );

    const attempt = await repairAndBuild(
      evalID,
      gateway,
      model,
      env,
      rootPromptDef,
      directory,
      finalOutputFiles,
      lastAttempt.buildResult.message,
      'There are the following build errors:',
      contextFiles,
      abortSignal,
      workerConcurrencyQueue,
      repairAttempts,
      progress
    );

    attemptDetails.push(attempt);
    lastAttempt = attempt;
  }

  // Now that we got a working app, try to serve it and collect
  // findings from the running app.
  lastAttempt.serveTestingResult = await serveAndTestApp(
    evalID,
    gateway,
    directory,
    env,
    rootPromptDef,
    workerConcurrencyQueue,
    abortSignal,
    progress,
    skipScreenshots,
    skipAxeTesting,
    enableAutoCsp,
    userJourneyAgentTaskInput
  );

  // Attempt to repair axe testing.
  let axeRepairAttempts = 0;
  while (
    lastAttempt.serveTestingResult &&
    (lastAttempt.serveTestingResult.axeViolations?.length ?? 0) > 0 &&
    axeRepairAttempts < maxRepairAttempts
  ) {
    axeRepairAttempts++;
    progress.log(
      rootPromptDef,
      'build',
      `Trying to repair axe accessibility violations (attempt #${axeRepairAttempts + 1})...`
    );

    const axeViolationsError = JSON.stringify(
      lastAttempt.serveTestingResult.axeViolations,
      null,
      2
    );

    progress.log(rootPromptDef, 'error', 'Found Axe accessibility violations');

    const attempt = await repairAndBuild(
      evalID,
      gateway,
      model,
      env,
      rootPromptDef,
      directory,
      finalOutputFiles,
      axeViolationsError,
      'There are the following accessibility errors from axe accessibility violations:',
      contextFiles,
      abortSignal,
      workerConcurrencyQueue,
      axeRepairAttempts + repairAttempts,
      progress
    );

    // Re-run serving & tests after Axe repair.
    // This allows us to check if we fixed the violations.
    attempt.serveTestingResult = await serveAndTestApp(
      evalID,
      gateway,
      directory,
      env,
      rootPromptDef,
      workerConcurrencyQueue,
      abortSignal,
      progress,
      skipScreenshots,
      skipAxeTesting,
      enableAutoCsp,
      userJourneyAgentTaskInput
    );

    attemptDetails.push(attempt);
    lastAttempt = attempt;

    if (attempt.serveTestingResult.axeViolations?.length === 0) {
      progress.log(
        rootPromptDef,
        'success',
        `Successfully fixed all Axe accessibility violations`
      );
    }
  }

  return {
    buildResult: lastAttempt.buildResult,
    serveTestingResult: lastAttempt.serveTestingResult,
    outputFiles: finalOutputFiles,
    repairAttempts,
    axeRepairAttempts,
  };
}