fix(evaluation): address code review issues

- Add Zod schema validation for layout judge response (prevents NaN from
  malformed model output)
- Add activity tracking to generateFormSpecWithLayout (matches generateFormSpec)
- Update formSpecGenerator type to accept activity-tracking params
- Add evaluationRunSchema.parse() before writing layout evaluation results
- Fix score() signature to include _groundTruth parameter per EvaluationKind interface
- Remove erroneous groundTruth filter in layout evaluation subcommand
- Export buildLayoutPrompt from form-documents public index
- Fix test import to use public index instead of internal path
- Remove buildLayoutJudgePrompt from evaluation public index (implementation detail)

Author: danielnaab

src/entrypoints/cli/commands/evaluate.ts

@@ -443,10 +443,9 @@ export async function evaluate(
         '../../../../fixtures/index'
       )
       const fixtures = loadAllFixturesForEvaluation()
-      const withGT = fixtures.filter((f) => f.groundTruth !== undefined)
 
-      if (withGT.length === 0) {
-        console.error('No fixtures with ground truth found.')
+      if (fixtures.length === 0) {
+        console.error('No fixtures found.')
         return 1
       }
 
@@ -482,12 +481,12 @@ export async function evaluate(
       )
 
       console.log(`Running layout evaluation: ${strategyMeta.metadata.name}`)
-      console.log(`Fixtures: ${withGT.length}`)
+      console.log(`Fixtures: ${fixtures.length}`)
 
       const start = Date.now()
       const cases: RunResult['cases'] = []
 
-      for (const fixture of withGT) {
+      for (const fixture of fixtures) {
         try {
           const result = await extractor.extract(fixture.pdf, {
             slug: fixture.slug,
@@ -527,6 +526,8 @@ export async function evaluate(
         cases,
       }
 
+      evaluationRunSchema.parse(runResult)
+
       mkdirSync(outDir, { recursive: true })
       const jsonPath = join(outDir, `${strategyId}.json`)
       writeFileSync(jsonPath, JSON.stringify(runResult, null, 2))

src/services/evaluation/index.ts

@@ -25,7 +25,5 @@ export {
 export { createLlmJudgeKind } from './kinds/pdf-field-extraction-judge'
 export { shapingCommandsKind } from './kinds/shaping-commands'
 export { createBedrockLayoutJudge } from './layout-judge'
-// Layout judge prompt
-export { buildLayoutJudgePrompt } from './layout-judge-prompt'
 export { evaluationRunSchema } from './schemas'
 export type { RunResult } from './types'

src/services/evaluation/kinds/layout-quality.ts

@@ -1,16 +1,15 @@
 import type { DataCollectionSpec } from '../../data-collection'
 import type { FormSpec } from '../../forms'
 import type { CaseMetrics, EvaluationKind, SummaryMetrics } from '../types'
+import type { LayoutJudgeResponse } from '../layout-judge-schemas'
+
+export type { LayoutJudgeResponse }
 
 export interface LayoutQualityOutput {
   spec: DataCollectionSpec
   formSpec: FormSpec
 }
 
-export interface LayoutJudgeResponse {
-  scores: Record<string, { score: number; rationale: string }>
-}
-
 export interface LayoutJudge {
   judge(
     spec: DataCollectionSpec,
@@ -41,7 +40,7 @@ export function createLayoutQualityKind(
     description:
       'Evaluates FormSpec layout quality using LLM-as-judge against a civic tech best practices rubric',
 
-    async score(output: LayoutQualityOutput): Promise<CaseMetrics> {
+    async score(output: LayoutQualityOutput, _groundTruth: undefined): Promise<CaseMetrics> {
       const response = await judge.judge(output.spec, output.formSpec)
 
       const metrics: Record<string, number> = {}

src/services/evaluation/layout-judge-schemas.ts

@@ -0,0 +1,12 @@
+import { z } from 'zod'
+
+const layoutJudgeEntrySchema = z.object({
+  score: z.number().min(1).max(5),
+  rationale: z.string(),
+})
+
+export const layoutJudgeResponseSchema = z.object({
+  scores: z.record(z.string(), layoutJudgeEntrySchema),
+})
+
+export type LayoutJudgeResponse = z.infer<typeof layoutJudgeResponseSchema>

src/services/evaluation/layout-judge.ts

@@ -3,8 +3,12 @@ import { fromNodeProviderChain } from '@aws-sdk/credential-providers'
 import { generateText } from 'ai'
 import type { DataCollectionSpec } from '../data-collection'
 import type { FormSpec } from '../forms'
-import type { LayoutJudge, LayoutJudgeResponse } from './kinds/layout-quality'
+import type { LayoutJudge } from './kinds/layout-quality'
 import { buildLayoutJudgePrompt } from './layout-judge-prompt'
+import {
+  type LayoutJudgeResponse,
+  layoutJudgeResponseSchema,
+} from './layout-judge-schemas'
 
 export function createBedrockLayoutJudge(model: string): LayoutJudge {
   const bedrock = createAmazonBedrock({
@@ -29,7 +33,7 @@ export function createBedrockLayoutJudge(model: string): LayoutJudge {
       const jsonStr = trimmed.startsWith('```')
         ? trimmed.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?```\s*$/, '')
         : trimmed
-      return JSON.parse(jsonStr) as LayoutJudgeResponse
+      return layoutJudgeResponseSchema.parse(JSON.parse(jsonStr))
     },
   }
 }

src/services/form-documents/extraction-steps.ts

@@ -91,12 +91,27 @@ ${JSON.stringify(spec, null, 2)}`,
 export async function generateFormSpecWithLayout(
   model: LanguageModel,
   spec: DataCollectionSpec,
+  activityStore?: ActivityStore,
+  userId?: string,
+  projectId?: string,
+  modelId?: string,
 ): Promise<FormSpec> {
+  const startTime = Date.now()
   const result = await generateText({
     model,
     maxOutputTokens: 8192,
     messages: [{ role: 'user', content: buildLayoutPrompt(spec) }],
   })
+  if (activityStore && modelId) {
+    trackLlmCall(activityStore, {
+      userId,
+      projectId,
+      operation: 'extraction-formspec-layout',
+      model: modelId,
+      usage: result.usage,
+      durationMs: Date.now() - startTime,
+    })
+  }
   return parseJsonResponse(result.text, formSpecSchema)
 }
 

src/services/form-documents/extraction.ts

@@ -113,11 +113,17 @@ export interface BedrockExtractorOptions {
   /**
    * Custom FormSpec generator for Step 2. When provided, replaces the
    * default `generateFormSpec` call. Use `generateFormSpecWithLayout`
-   * for layout-aware generation.
+   * for layout-aware generation. Receives the same activity-tracking
+   * arguments as `generateFormSpec` so LLM cost is tracked regardless
+   * of which generator is active.
    */
   formSpecGenerator?: (
     model: LanguageModel,
     spec: DataCollectionSpec,
+    activityStore?: ActivityStore,
+    userId?: string,
+    projectId?: string,
+    modelId?: string,
   ) => Promise<FormSpec>
 }
 
@@ -324,7 +330,14 @@ ${exemplarSection}Guidelines:
       // Step 2: Generate default FormSpec from extracted spec
       const bedrockModel = bedrock(model)
       const formSpec = options?.formSpecGenerator
-        ? await options.formSpecGenerator(bedrockModel, spec)
+        ? await options.formSpecGenerator(
+            bedrockModel,
+            spec,
+            options?.activityStore,
+            extractionOptions?.userId,
+            extractionOptions?.slug,
+            model,
+          )
         : await generateFormSpec(
             bedrockModel,
             spec,

src/services/form-documents/index.ts

@@ -8,6 +8,7 @@ export {
   createCachedPdfExtractor,
 } from './extraction'
 export { generateFormSpecWithLayout } from './extraction-steps'
+export { buildLayoutPrompt } from './layout-prompt'
 export { enumerateFields } from './field-mapping'
 export { fillPdf } from './filling'
 export { createMappingRegistry } from './mapping-registry'

test/form-documents/layout-prompt.test.ts

@@ -1,7 +1,9 @@
 import { describe, expect, it } from 'bun:test'
 import type { DataCollectionSpec } from '../../src/services/data-collection'
-import { generateFormSpecWithLayout } from '../../src/services/form-documents/extraction-steps'
-import { buildLayoutPrompt } from '../../src/services/form-documents/layout-prompt'
+import {
+  buildLayoutPrompt,
+  generateFormSpecWithLayout,
+} from '../../src/services/form-documents'
 
 const smallSpec: DataCollectionSpec = {
   id: 'contact-info',