Add prompt-improvement briefs to eval loop

Author: MaxLinCode

.gitignore

@@ -10,3 +10,4 @@ coverage
 *.tsbuildinfo
 packages/integrations/*.manual-eval-report.json
 packages/integrations/manual-eval-report.json
+packages/integrations/*.prompt-improvement.md

README.md

@@ -32,6 +32,33 @@ Atlas is a chat-first brain-dump scheduler MVP. A user sends freeform text, the
 - `pnpm db:test:reset`: reset the local `atlas_test` database schema for integration reruns
 - `pnpm db:test:stop`: stop the local Homebrew Postgres test service
 
+## Prompt Improvement Loop
+
+Atlas treats prompt changes as product behavior changes. Use the live eval harness to iterate on prompts deliberately instead of editing blind.
+
+Recommended loop:
+
+1. Edit the owning prompt, schema, or parser in `packages/integrations` and `packages/core` together when the contract changes.
+2. Run the narrowest relevant live eval first:
+   - `pnpm eval:planner`
+   - `pnpm eval:turn-router`
+   - `pnpm eval:router-confirmation`
+   - `pnpm eval:conversation-context`
+   - `pnpm eval:confirmed-mutation-recovery`
+3. Inspect the suite-specific report written under `packages/integrations/*.manual-eval-report.json`.
+4. If the suite fails, inspect the generated prompt-improvement brief under `packages/integrations/*.prompt-improvement.md` and use it as the starting point for the next prompt revision.
+5. Tighten the prompt or contract based on the actual failing model output.
+6. Run `pnpm eval:all` before merging to confirm the full prompt surface still passes together.
+
+Notes:
+
+- `pnpm eval:all` writes the canonical consolidated report to `packages/integrations/manual-eval-report.json`.
+- Single-suite evals write suite-specific reports such as `packages/integrations/conversation-context.manual-eval-report.json`.
+- Failing suites also write suite-specific prompt-improvement briefs such as `packages/integrations/conversation-context.prompt-improvement.md`.
+- Generated eval reports are ignored by git and should not be committed.
+- Generated prompt-improvement briefs are also ignored by git and should not be committed.
+- Live evals help judge prompt quality, but they do not replace deterministic tests and schema validation.
+
 ## Local Integration DB
 
 Atlas supports an explicit local Postgres workflow for integration tests on macOS with Homebrew `postgresql@16`.

packages/integrations/src/manual/all.eval.test.ts

@@ -4,7 +4,12 @@ import { runConfirmedMutationRecoveryEvalSuite } from "./confirmed-mutation-reco
 import { runConversationContextEvalSuite } from "./conversation-context.eval-suite";
 import { runPlannerEvalSuite } from "./planner.eval-suite";
 import { runRouterConfirmationEvalSuite } from "./router-confirmation.eval-suite";
-import { ensureManualEvalEnv, buildEvalReport, writeEvalReport } from "./shared";
+import {
+  ensureManualEvalEnv,
+  buildEvalReport,
+  writeEvalReport,
+  writePromptImprovementBriefsForFailures
+} from "./shared";
 import { runTurnRouterEvalSuite } from "./turn-router.eval-suite";
 
 beforeAll(() => {
@@ -23,6 +28,7 @@ describe.sequential("manual prompt eval loop", () => {
 
     const report = buildEvalReport(suites);
     const reportPath = await writeEvalReport(report);
+    const briefPaths = await writePromptImprovementBriefsForFailures(suites);
 
     console.table(
       suites.map((suite) => ({
@@ -34,6 +40,9 @@ describe.sequential("manual prompt eval loop", () => {
       }))
     );
     console.log(`Manual eval report written to ${reportPath}`);
+    if (briefPaths.length > 0) {
+      console.log(`Prompt improvement briefs written to ${briefPaths.join(", ")}`);
+    }
 
     expect(report.failedCases).toBe(0);
   }, 300_000);

packages/integrations/src/manual/confirmed-mutation-recovery.eval.test.ts

@@ -1,7 +1,11 @@
 import { beforeAll, describe, expect, it } from "vitest";
 
 import { runConfirmedMutationRecoveryEvalSuite } from "./confirmed-mutation-recovery.eval-suite";
-import { ensureManualEvalEnv } from "./shared";
+import {
+  ensureManualEvalEnv,
+  writePromptImprovementBrief,
+  writeSuiteEvalReport
+} from "./shared";
 
 beforeAll(() => {
   ensureManualEvalEnv();
@@ -10,6 +14,9 @@ beforeAll(() => {
 describe.sequential("manual confirmed-mutation recovery eval", () => {
   it("checks curated recovery cases against the live OpenAI prompt", async () => {
     const suite = await runConfirmedMutationRecoveryEvalSuite();
+    const reportPath = await writeSuiteEvalReport(suite);
+    const briefPath =
+      suite.failed > 0 ? await writePromptImprovementBrief(suite) : null;
 
     console.table(
       suite.cases.map((testCase) => ({
@@ -23,6 +30,10 @@ describe.sequential("manual confirmed-mutation recovery eval", () => {
         error: testCase.error ?? ""
       }))
     );
+    console.log(`Manual eval report written to ${reportPath}`);
+    if (briefPath) {
+      console.log(`Prompt improvement brief written to ${briefPath}`);
+    }
 
     expect(suite.failed).toBe(0);
   }, 120_000);

packages/integrations/src/manual/conversation-context.eval.test.ts

@@ -1,7 +1,11 @@
 import { beforeAll, describe, expect, it } from "vitest";
 
 import { runConversationContextEvalSuite } from "./conversation-context.eval-suite";
-import { ensureManualEvalEnv, writeSuiteEvalReport } from "./shared";
+import {
+  ensureManualEvalEnv,
+  writePromptImprovementBrief,
+  writeSuiteEvalReport
+} from "./shared";
 
 beforeAll(() => {
   ensureManualEvalEnv();
@@ -11,6 +15,8 @@ describe.sequential("manual conversation context eval", () => {
   it("checks curated continuity cases against the live OpenAI prompts", async () => {
     const suite = await runConversationContextEvalSuite();
     const reportPath = await writeSuiteEvalReport(suite);
+    const briefPath =
+      suite.failed > 0 ? await writePromptImprovementBrief(suite) : null;
 
     console.table(
       suite.cases.map((testCase) => ({
@@ -23,6 +29,9 @@ describe.sequential("manual conversation context eval", () => {
       }))
     );
     console.log(`Manual eval report written to ${reportPath}`);
+    if (briefPath) {
+      console.log(`Prompt improvement brief written to ${briefPath}`);
+    }
 
     expect(suite.failed).toBe(0);
   }, 120_000);

packages/integrations/src/manual/planner.eval.test.ts

@@ -1,7 +1,11 @@
 import { beforeAll, describe, expect, it } from "vitest";
 
 import { runPlannerEvalSuite } from "./planner.eval-suite";
-import { ensureManualEvalEnv } from "./shared";
+import {
+  ensureManualEvalEnv,
+  writePromptImprovementBrief,
+  writeSuiteEvalReport
+} from "./shared";
 
 beforeAll(() => {
   ensureManualEvalEnv();
@@ -10,6 +14,9 @@ beforeAll(() => {
 describe.sequential("manual planner eval", () => {
   it("checks curated planner timing cases against the live OpenAI prompt", async () => {
     const suite = await runPlannerEvalSuite();
+    const reportPath = await writeSuiteEvalReport(suite);
+    const briefPath =
+      suite.failed > 0 ? await writePromptImprovementBrief(suite) : null;
 
     console.table(
       suite.cases.map((testCase) => ({
@@ -22,6 +29,10 @@ describe.sequential("manual planner eval", () => {
         error: testCase.error ?? ""
       }))
     );
+    console.log(`Manual eval report written to ${reportPath}`);
+    if (briefPath) {
+      console.log(`Prompt improvement brief written to ${briefPath}`);
+    }
 
     expect(suite.failed).toBe(0);
   }, 120_000);

packages/integrations/src/manual/router-confirmation.eval.test.ts

@@ -1,7 +1,11 @@
 import { beforeAll, describe, expect, it } from "vitest";
 
 import { runRouterConfirmationEvalSuite } from "./router-confirmation.eval-suite";
-import { ensureManualEvalEnv } from "./shared";
+import {
+  ensureManualEvalEnv,
+  writePromptImprovementBrief,
+  writeSuiteEvalReport
+} from "./shared";
 
 beforeAll(() => {
   ensureManualEvalEnv();
@@ -10,6 +14,9 @@ beforeAll(() => {
 describe.sequential("manual router confirmation eval", () => {
   it("checks curated confirmation cases against the live OpenAI prompt", async () => {
     const suite = await runRouterConfirmationEvalSuite();
+    const reportPath = await writeSuiteEvalReport(suite);
+    const briefPath =
+      suite.failed > 0 ? await writePromptImprovementBrief(suite) : null;
 
     console.table(
       suite.cases.map((testCase) => ({
@@ -22,6 +29,10 @@ describe.sequential("manual router confirmation eval", () => {
         error: testCase.error ?? ""
       }))
     );
+    console.log(`Manual eval report written to ${reportPath}`);
+    if (briefPath) {
+      console.log(`Prompt improvement brief written to ${briefPath}`);
+    }
 
     expect(suite.failed).toBe(0);
   }, 120_000);

packages/integrations/src/manual/shared.ts

@@ -1,6 +1,12 @@
 import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 
+import { confirmedMutationRecoverySystemPrompt } from "../prompts/confirmed-mutation-recovery";
+import { conversationMemorySummarySystemPrompt } from "../prompts/conversation-memory-summary";
+import { conversationResponseSystemPrompt } from "../prompts/conversation-response";
+import { inboxPlannerSystemPrompt } from "../prompts/planner";
+import { turnRouterSystemPrompt } from "../prompts/turn-router";
+
 export type EvalCaseResult = {
   name: string;
   pass: boolean;
@@ -26,6 +32,20 @@ export type EvalReport = {
   suites: EvalSuiteResult[];
 };
 
+const SUITE_PROMPTS: Record<string, string> = {
+  planner: inboxPlannerSystemPrompt,
+  "turn-router": turnRouterSystemPrompt,
+  "router-confirmation": turnRouterSystemPrompt,
+  "confirmed-mutation-recovery": confirmedMutationRecoverySystemPrompt,
+  "conversation-context": [
+    "Conversation Response Prompt:",
+    conversationResponseSystemPrompt,
+    "",
+    "Conversation Memory Summary Prompt:",
+    conversationMemorySummarySystemPrompt
+  ].join("\n")
+};
+
 export function ensureManualEvalEnv() {
   if (!process.env.OPENAI_API_KEY?.trim()) {
     throw new Error("OPENAI_API_KEY is required to run the manual evals.");
@@ -78,3 +98,102 @@ export async function writeSuiteEvalReport(suite: EvalSuiteResult) {
 
   return reportPath;
 }
+
+function formatCaseDetails(details: Record<string, unknown>) {
+  return Object.entries(details)
+    .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
+    .join("\n");
+}
+
+function buildPromptImprovementPrompt(suite: EvalSuiteResult) {
+  const originalPrompt =
+    SUITE_PROMPTS[suite.suiteName] ??
+    "Prompt text unavailable for this suite. Inspect the owning prompt module directly.";
+
+  const failedCases = suite.cases.filter((testCase) => !testCase.pass);
+  const testResults =
+    failedCases.length === 0
+      ? "No failing cases."
+      : failedCases
+          .map(
+            (testCase) =>
+              [
+                `Case: ${testCase.name}`,
+                `Details:`,
+                formatCaseDetails(testCase.details),
+                testCase.error ? `Error: ${testCase.error}` : null
+              ]
+                .filter(Boolean)
+                .join("\n")
+          )
+          .join("\n\n---\n\n");
+
+  return [
+    "You are optimizing a production prompt to improve eval pass rate without breaking its intended behavior.",
+    "",
+    "You will receive:",
+    "- `Original prompt`: the current prompt text",
+    "- `Test results`: failing cases, logs, model outputs, and any passing-context clues",
+    "",
+    "Your job:",
+    "1. Diagnose why the prompt underperformed.",
+    "2. Identify the smallest prompt changes likely to improve behavior.",
+    "3. Rewrite the prompt to improve test performance while preserving the original contract.",
+    "4. Avoid brittle edits that only patch one example.",
+    "",
+    "Working rules:",
+    "- Preserve the original intent, role, and task boundary.",
+    "- Prefer minimal but high-leverage edits.",
+    "- Do not overfit to one failure string.",
+    "- Generalize from the failure pattern.",
+    "- Keep successful existing behavior intact unless the failures show a real conflict.",
+    "- If the test failure suggests a schema or evaluator problem rather than a prompt problem, say so briefly before proposing prompt edits.",
+    "",
+    "Output exactly in this format:",
+    "",
+    "Improved prompt:",
+    "\"\"\"",
+    "<full revised prompt>",
+    "\"\"\"",
+    "",
+    "Explanation:",
+    "- <short bullet explaining the main failure pattern>",
+    "- <short bullet explaining the key prompt changes>",
+    "- <short bullet explaining why the revision should generalize better>",
+    "",
+    "Original prompt:",
+    "\"\"\"",
+    originalPrompt,
+    "\"\"\"",
+    "",
+    "Test results:",
+    "\"\"\"",
+    testResults,
+    "\"\"\""
+  ].join("\n");
+}
+
+export async function writePromptImprovementBrief(suite: EvalSuiteResult) {
+  const briefPath =
+    process.env.ATLAS_PROMPT_IMPROVEMENT_PATH ??
+    path.resolve(process.cwd(), `${suite.suiteName}.prompt-improvement.md`);
+
+  await mkdir(path.dirname(briefPath), {
+    recursive: true
+  });
+  await writeFile(briefPath, buildPromptImprovementPrompt(suite), "utf8");
+
+  return briefPath;
+}
+
+export async function writePromptImprovementBriefsForFailures(
+  suites: EvalSuiteResult[]
+) {
+  const failedSuites = suites.filter((suite) => suite.failed > 0);
+
+  const briefPaths = await Promise.all(
+    failedSuites.map((suite) => writePromptImprovementBrief(suite))
+  );
+
+  return briefPaths;
+}

packages/integrations/src/manual/turn-router.eval.test.ts

@@ -1,6 +1,10 @@
 import { beforeAll, describe, expect, it } from "vitest";
 
-import { ensureManualEvalEnv } from "./shared";
+import {
+  ensureManualEvalEnv,
+  writePromptImprovementBrief,
+  writeSuiteEvalReport
+} from "./shared";
 import { runTurnRouterEvalSuite } from "./turn-router.eval-suite";
 
 beforeAll(() => {
@@ -10,6 +14,9 @@ beforeAll(() => {
 describe.sequential("manual turn router eval", () => {
   it("checks curated routing cases against the live OpenAI prompt", async () => {
     const suite = await runTurnRouterEvalSuite();
+    const reportPath = await writeSuiteEvalReport(suite);
+    const briefPath =
+      suite.failed > 0 ? await writePromptImprovementBrief(suite) : null;
 
     console.table(
       suite.cases.map((testCase) => ({
@@ -22,6 +29,10 @@ describe.sequential("manual turn router eval", () => {
         error: testCase.error ?? ""
       }))
     );
+    console.log(`Manual eval report written to ${reportPath}`);
+    if (briefPath) {
+      console.log(`Prompt improvement brief written to ${briefPath}`);
+    }
 
     expect(suite.failed).toBe(0);
   }, 60_000);