Add Atlas eval loop and repo skills

Author: MaxLinCode

.gitignore

@@ -8,3 +8,5 @@ coverage
 .env.local
 .env.*.local
 *.tsbuildinfo
+packages/integrations/*.manual-eval-report.json
+packages/integrations/manual-eval-report.json

README.md

@@ -15,6 +15,7 @@ Atlas is a chat-first brain-dump scheduler MVP. A user sends freeform text, the
 
 - `pnpm dev`: run the Next.js app locally
 - `pnpm build`: build all workspaces
+- `pnpm eval:all`: run the full live OpenAI prompt-eval loop and write a consolidated report to `packages/integrations/manual-eval-report.json`
 - `pnpm eval:confirmed-mutation-recovery`: run the live OpenAI confirmed-mutation recovery eval fixture set against the current prompt
 - `pnpm eval:conversation-context`: run the live OpenAI conversation-context eval fixture set against the current prompt
 - `pnpm eval:planner`: run the live OpenAI planner eval fixture set against the current prompt
@@ -88,6 +89,22 @@ This repo is designed for human-plus-agent collaboration.
 - Update `docs/current-work.md` when priorities or handoff context changes.
 - Capture architectural changes in `docs/decisions/`.
 
+## Repo-local Skills
+
+Atlas includes repo-specific Codex skills under [`skills/`](./skills). These are versioned with the repo so they can evolve with Atlas architecture, workflow rules, and testing expectations rather than living as global defaults.
+
+Current skills:
+
+- `atlas-feature-delivery`
+- `atlas-planning-change`
+- `atlas-webhook-and-conversation`
+- `atlas-schema-and-migration`
+- `atlas-openai-contracts-and-evals`
+- `atlas-google-calendar-flow`
+- `atlas-reviewer`
+
+Use these when working inside Atlas and the task matches the skill name. Keep the repo copy as the source of truth.
+
 ## MVP flow
 
 1. The current messaging webhook receives a freeform message.

apps/web/src/lib/server/process-inbox-item.test.ts

@@ -1189,6 +1189,43 @@ describe("process inbox item service", () => {
     );
   });
 
+  it("does not leak internal planner validation reasons to the user", async () => {
+    seedInboxItemForProcessingTests({
+      id: "inbox-internal-reason",
+      userId: "123",
+      sourceEventId: "event-internal-reason",
+      rawText: "Pick an open spot for that",
+      normalizedText: "Pick an open spot for that",
+      processingStatus: "received",
+      linkedTaskIds: []
+    });
+
+    const result = await processInboxItem(
+      { inboxItemId: "inbox-internal-reason" },
+      {
+        calendar: getDefaultCalendarAdapter(),
+        planner: async () => ({
+          confidence: 0.25,
+          summary: "Need clarification.",
+          actions: [
+            {
+              type: "create_task",
+              alias: "new_task_1",
+              title: "Oil change",
+              priority: "medium",
+              urgency: "medium"
+            }
+          ]
+        })
+      }
+    );
+
+    expect(result.outcome).toBe("needs_clarification");
+    expect(result.followUpMessage).toBe(
+      "I couldn't safely apply that update. Tell me the exact task and what you'd like me to change."
+    );
+  });
+
   it("asks for clarification when multiple existing tasks share the same title", async () => {
     const store = getDefaultInboxProcessingStore();
 

apps/web/src/lib/server/process-inbox-item.ts

@@ -822,15 +822,30 @@ function saveClarification(input: ApplyPlanningResultInput, reason: string) {
 }
 
 function buildClarificationReply(reason: string) {
+  if (!isSafeUserFacingClarificationReason(reason)) {
+    return "I couldn't safely apply that update. Tell me the exact task and what you'd like me to change.";
+  }
+
+  return reason;
+}
+
+function isSafeUserFacingClarificationReason(reason: string) {
   if (
     reason.startsWith("Model returned") ||
     reason.startsWith("Could not resolve") ||
-    reason.startsWith("Expected ")
+    reason.startsWith("Expected ") ||
+    reason.startsWith("Each created task must") ||
+    reason.startsWith("Model did not provide") ||
+    reason.startsWith("Please connect Google Calendar before") ||
+    reason.startsWith("The linked Google Calendar event changed outside Atlas.") ||
+    reason.startsWith("I couldn't safely apply that update.")
   ) {
-    return "I couldn't safely apply that update. Tell me the exact task and what you'd like me to change.";
+    return reason.startsWith("Please connect Google Calendar before") ||
+      reason.startsWith("The linked Google Calendar event changed outside Atlas.") ||
+      reason.startsWith("I couldn't safely apply that update.");
   }
 
-  return reason;
+  return true;
 }
 
 function hasAmbiguousTaskTitle(tasks: Task[], task: Task) {

docs/current-work.md

@@ -3,7 +3,7 @@
 ## Active focus
 
 Atlas is a schedule-forward, Google-calendar-gated product with a working mutation pipeline.
-Current implementation focus: harden the model-driven planning layer now that the Google Calendar path is locked down. The active branch restructures the OpenAI prompts into explicit prompt assets, expands live eval coverage around ambiguous routing and confirmed-mutation recovery, shifts product language from Telegram-first to chat-first at the prompt and docs layer, and threads `referenceTime` consistently through scheduling so temporal interpretation and busy-calendar lookups use the same anchor.
+Current implementation focus: harden the model-driven planning layer now that the Google Calendar path is locked down. The active branch restructures the OpenAI prompts into explicit prompt assets, expands live eval coverage around ambiguous routing and confirmed-mutation recovery, adds a consolidated `pnpm eval:all` loop plus suite-specific eval reports for prompt iteration, shifts product language from Telegram-first to chat-first at the prompt and docs layer, and threads `referenceTime` consistently through scheduling so temporal interpretation and busy-calendar lookups use the same anchor. Atlas also now carries repo-local Codex skills under `skills/` so workflow guidance can evolve with the codebase instead of living only in global agent defaults.
 
 ## Near-term milestones
 

package.json

@@ -5,6 +5,7 @@
   "scripts": {
     "dev": "pnpm --filter @atlas/web dev",
     "build": "pnpm -r build",
+    "eval:all": "pnpm --filter @atlas/integrations eval:all",
     "eval:confirmed-mutation-recovery": "pnpm --filter @atlas/integrations eval:confirmed-mutation-recovery",
     "eval:conversation-context": "pnpm --filter @atlas/integrations eval:conversation-context",
     "eval:planner": "pnpm --filter @atlas/integrations eval:planner",

packages/integrations/package.json

@@ -8,6 +8,7 @@
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",
+    "eval:all": "vitest run --config vitest.manual.config.ts src/manual/all.eval.test.ts",
     "eval:confirmed-mutation-recovery": "vitest run --config vitest.manual.config.ts src/manual/confirmed-mutation-recovery.eval.test.ts",
     "eval:conversation-context": "vitest run --config vitest.manual.config.ts src/manual/conversation-context.eval.test.ts",
     "eval:planner": "vitest run --config vitest.manual.config.ts src/manual/planner.eval.test.ts",

packages/integrations/src/manual/all.eval.test.ts

@@ -0,0 +1,40 @@
+import { beforeAll, describe, expect, it } from "vitest";
+
+import { runConfirmedMutationRecoveryEvalSuite } from "./confirmed-mutation-recovery.eval-suite";
+import { runConversationContextEvalSuite } from "./conversation-context.eval-suite";
+import { runPlannerEvalSuite } from "./planner.eval-suite";
+import { runRouterConfirmationEvalSuite } from "./router-confirmation.eval-suite";
+import { ensureManualEvalEnv, buildEvalReport, writeEvalReport } from "./shared";
+import { runTurnRouterEvalSuite } from "./turn-router.eval-suite";
+
+beforeAll(() => {
+  ensureManualEvalEnv();
+});
+
+describe.sequential("manual prompt eval loop", () => {
+  it("runs all live prompt eval suites and writes one report", async () => {
+    const suites = [
+      await runPlannerEvalSuite(),
+      await runTurnRouterEvalSuite(),
+      await runRouterConfirmationEvalSuite(),
+      await runConversationContextEvalSuite(),
+      await runConfirmedMutationRecoveryEvalSuite()
+    ];
+
+    const report = buildEvalReport(suites);
+    const reportPath = await writeEvalReport(report);
+
+    console.table(
+      suites.map((suite) => ({
+        suite: suite.suiteName,
+        total: suite.total,
+        passed: suite.passed,
+        failed: suite.failed,
+        durationMs: suite.durationMs
+      }))
+    );
+    console.log(`Manual eval report written to ${reportPath}`);
+
+    expect(report.failedCases).toBe(0);
+  }, 300_000);
+});

packages/integrations/src/manual/confirmed-mutation-recovery.eval-suite.ts

@@ -0,0 +1,136 @@
+import type { ConfirmedMutationRecoveryInput } from "@atlas/core";
+import { expect } from "vitest";
+
+import { recoverConfirmedMutationWithResponses } from "../openai";
+import type { EvalCaseResult, EvalSuiteResult } from "./shared";
+
+type RecoveryEvalCase = {
+  name: string;
+  input: ConfirmedMutationRecoveryInput;
+  assert: (result: Awaited<ReturnType<typeof recoverConfirmedMutationWithResponses>>) => void;
+};
+
+export const RECOVERY_EVAL_CASES: RecoveryEvalCase[] = [
+  {
+    name: "single concrete confirmation recovers one write-ready request",
+    input: {
+      rawText: "Yes",
+      normalizedText: "Yes",
+      recentTurns: [
+        {
+          role: "assistant",
+          text: "Would you like me to schedule it at 3pm?",
+          createdAt: "2026-03-17T16:00:00.000Z"
+        },
+        {
+          role: "user",
+          text: "Yes",
+          createdAt: "2026-03-17T16:01:00.000Z"
+        }
+      ],
+      memorySummary: "The assistant proposed scheduling it at 3pm."
+    },
+    assert: (result) => {
+      expect(result.outcome).toBe("recovered");
+      expect(result.recoveredText).toMatch(/3pm/i);
+    }
+  },
+  {
+    name: "vague yes after multiple options requires clarification",
+    input: {
+      rawText: "Yes",
+      normalizedText: "Yes",
+      recentTurns: [
+        {
+          role: "assistant",
+          text: "I could do 3pm or 4pm.",
+          createdAt: "2026-03-17T16:00:00.000Z"
+        },
+        {
+          role: "user",
+          text: "Yes",
+          createdAt: "2026-03-17T16:01:00.000Z"
+        }
+      ],
+      memorySummary: "Two candidate times were proposed."
+    },
+    assert: (result) => {
+      expect(result.outcome).toBe("needs_clarification");
+      expect(result.recoveredText).toBeNull();
+      expect(result.userReplyMessage).toMatch(/\?/);
+    }
+  },
+  {
+    name: "clear completion language recovers a completion request",
+    input: {
+      rawText: "done",
+      normalizedText: "done",
+      recentTurns: [
+        {
+          role: "assistant",
+          text: "Did you finish the journaling session?",
+          createdAt: "2026-03-17T16:00:00.000Z"
+        },
+        {
+          role: "user",
+          text: "done",
+          createdAt: "2026-03-17T16:01:00.000Z"
+        }
+      ],
+      memorySummary: "The recent exchange is about the journaling session."
+    },
+    assert: (result) => {
+      expect(result.outcome).toBe("recovered");
+      expect(result.recoveredText).toMatch(/journal|done/i);
+    }
+  }
+];
+
+export async function runConfirmedMutationRecoveryEvalSuite(): Promise<EvalSuiteResult> {
+  const startedAt = Date.now();
+  const cases: EvalCaseResult[] = [];
+
+  for (const testCase of RECOVERY_EVAL_CASES) {
+    const result = await recoverConfirmedMutationWithResponses(testCase.input);
+
+    try {
+      testCase.assert(result);
+      cases.push({
+        name: testCase.name,
+        pass: true,
+        details: {
+          input: testCase.input.rawText,
+          outcome: result.outcome,
+          recoveredText: result.recoveredText,
+          reason: result.reason,
+          userReplyMessage: result.userReplyMessage
+        }
+      });
+    } catch (error) {
+      cases.push({
+        name: testCase.name,
+        pass: false,
+        details: {
+          input: testCase.input.rawText,
+          outcome: result.outcome,
+          recoveredText: result.recoveredText,
+          reason: result.reason,
+          userReplyMessage: result.userReplyMessage
+        },
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+
+  const passed = cases.filter((testCase) => testCase.pass).length;
+
+  return {
+    suiteName: "confirmed-mutation-recovery",
+    total: cases.length,
+    passed,
+    failed: cases.length - passed,
+    durationMs: Date.now() - startedAt,
+    cases
+  };
+}
+