feat: expose helper for writing reports to disk

This is useful for programmatic usage of WCS.

Author: devversion

runner/eval-cli.ts

@@ -7,6 +7,7 @@ import {
   DEFAULT_MAX_TEST_REPAIR_ATTEMPTS,
   DEFAULT_MODEL_NAME,
   DEFAULT_PROMPT_TIMEOUT_RETRIES,
+  REPORTS_ROOT_DIR,
 } from './configuration/constants.js';
 import {generateCodeAndAssess} from './orchestration/generate.js';
 import {logReportToConsole, writeReportToDisk} from './reporting/report-logging.js';
@@ -236,7 +237,7 @@ async function handler(cliArgs: Arguments<Options>): Promise<void> {
     });
 
     logReportToConsole(runInfo);
-    await writeReportToDisk(runInfo, runInfo.details.summary.environmentId);
+    await writeReportToDisk(runInfo, runInfo.details.summary.environmentId, REPORTS_ROOT_DIR);
   } catch (error: unknown) {
     if (error instanceof UserFacingError) {
       console.error(chalk.red(error.message));

runner/index.ts

@@ -51,3 +51,4 @@ export {type ServeTestingResult} from './workers/serve-testing/worker-types.js';
 export {replaceAtReferencesInPrompt} from './utils/prompt-at-references.js';
 export {extractRubrics} from './utils/extract-rubrics.js';
 export {combineReports} from './utils/combine-reports.mjs';
+export {writeReportToDisk} from './reporting/report-logging.js';

runner/reporting/report-logging.ts

@@ -2,7 +2,7 @@ import {join} from 'path';
 import chalk from 'chalk';
 import boxen from 'boxen';
 import {IndividualAssessmentState, RunInfo, ScoreBucket} from '../shared-interfaces.js';
-import {DEFAULT_AUTORATER_MODEL_NAME, REPORTS_ROOT_DIR} from '../configuration/constants.js';
+import {DEFAULT_AUTORATER_MODEL_NAME} from '../configuration/constants.js';
 import {calculateBuildAndCheckStats} from '../ratings/stats.js';
 import {safeWriteFile} from '../file-system-utils.js';
 import {BuildResultStatus} from '../workers/builder/builder-types.js';
@@ -17,7 +17,6 @@ import {
 } from './format.js';
 import {Environment} from '../configuration/environment.js';
 import {groupSimilarReports} from '../orchestration/grouping.js';
-import {LocalExecutor} from '../orchestration/executors/local-executor.js';
 
 /**
  * Generates a structured report on fs, based on the assessment run information.
@@ -38,14 +37,19 @@ import {LocalExecutor} from '../orchestration/executors/local-executor.js';
  *
  * @param runInfo An object containing all details and results of the assessment run.
  * @param id ID of the environment that was used for the eval.
+ * @param reportsRootDir Root directory where the reports are written to.
  * @returns The original `runInfo` object, allowing for chaining.
  */
-export async function writeReportToDisk(runInfo: RunInfo, id: string): Promise<void> {
+export async function writeReportToDisk(
+  runInfo: RunInfo,
+  id: string,
+  reportsRootDir: string,
+): Promise<void> {
   // Sanitize report name: allow only a-z, A-Z, 0-9, and hyphens. Replace others with a hyphen.
   const sanitizedReportName = runInfo.details.reportName.replace(/[^a-zA-Z0-9-]/g, '-');
 
   const {results} = runInfo;
-  const reportBaseDir = join(REPORTS_ROOT_DIR, id, sanitizedReportName);
+  const reportBaseDir = join(reportsRootDir, id, sanitizedReportName);
 
   // Write `summary.json` file, which contains **all** available info.
   const summaryJsonPath = join(reportBaseDir, 'summary.json');