refactor: pass root prompt to executor#initializeEval

This allows eval IDs to be based on prompt names/IDs. This is useful for
e.g. supporting pre-scrape executors and mapping based on eval IDs to
pre-sampled outputs.

Author: devversion

runner/orchestration/executors/executor.ts

@@ -23,7 +23,10 @@ export type EvalID = string & {__evalID: true};
 export type WorkerQueueType = PQueue;
 
 export const executorSchema = z.object({
-  initializeEval: z.function(z.tuple([]), z.promise(z.custom<EvalID>())),
+  initializeEval: z.function(
+    z.tuple([z.custom<RootPromptDefinition>()]),
+    z.promise(z.custom<EvalID>()),
+  ),
   generateInitialFiles: z.function(
     z.tuple([
       z.custom<EvalID>().describe('ID of the eval'),

runner/orchestration/generate.ts

@@ -158,7 +158,7 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
       allTasks.push(
         appConcurrencyQueue.add(async () => {
           const evaluate = async () => {
-            const evalID = await env.executor.initializeEval();
+            const evalID = await env.executor.initializeEval(rootPromptDef);
             let results: AssessmentResult[] | undefined;
 
             try {