feat: add validation framework with conversation logger

Bharatram-altimate-ai · claude · Bharatram-altimate-ai · commit 2a2227cf4b26 · 2026-03-13T16:51:22.000+05:30
Adds the /validate skill, conversation logging, and associated tooling.

**Validation framework**
- Add `src/skill/validate/SKILL.md` — 7-step validation workflow:
  steps 1–5 fetch and analyse traces, step 6 generates an HTML dashboard,
  step 7 categorises groundedness failures
- Add `src/skill/validate/batch_validate.py` — calls Altimate backend
  SSE endpoints (`/validate`, `/validate/session`, `/validate/date-range`)
  and writes structured JSON results; hardcoded prod URL and token, no
  dotenv dependency
- Embed both files at build time via `ALTIMATE_VALIDATE_SKILL_MD` and
  `ALTIMATE_VALIDATE_BATCH_PY` defines in `build.ts`
- Replace LLM-based intent classifier with deterministic `/validate`
  prefix check via `resolvePrompt` helper in `session.ts`, covering both
  `prompt` and `prompt_async` routes
- Delete `src/altimate/intent/validation-classifier.ts` and its directory

**Conversation logger**
- Add `src/session/conversation-logger.ts` — logs the last user+assistant
  turn to `POST /log-conversation` on session idle; collects up to 500
  messages to capture multi-step sessions; fire-and-forget, never blocks
- Gate logger in `bootstrap.ts` behind `ALTIMATE_LOGGER_DISABLED=true`
- Add `docs/docs/configure/logging.md` documenting what is collected
- Add Data Collection section to `README.md` with opt-out instructions

**Build / cleanup**
- Remove duplicate `ALTIMATE_CLI_*` constants from `build.ts`
- Remove `load_dotenv` and dead path-resolution from `batch_validate.py`
- Remove Langfuse reference from `SKILL.md` comment
- Update `uv run` invocations in `SKILL.md` to drop `--with python-dotenv`

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/.gitignore b/.gitignore
@@ -23,8 +23,37 @@ target
 .scripts
 .direnv/
 
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+
+# SQLite databases (feedback store creates these at runtime)
+*.db
+
+# Runtime logs
+*.log
+logs/
+
+# Large intermediate files at repo root (generated during benchmark runs)
+/queries.json
+/queries_1k.json
+/results/
+
+# Local runtime config
+.altimate-code/
+
+# Commit message scratch files
+.github/meta/
+
+# Experiment / simulation artifacts
+/data/
+/experiments/
+/models/
+/simulation/
+
 # Local dev files
 opencode-dev
-logs/
 *.bun-build
 tsconfig.tsbuildinfo
diff --git a/README.md b/README.md
@@ -157,6 +157,20 @@ Full docs at **[altimate-code.sh](https://altimate-code.sh)**.
 - [Agent Modes](https://altimate-code.sh/data-engineering/agent-modes/)
 - [Configuration](https://altimate-code.sh/configure/model-providers/)
 
+## Data Collection
+
+Altimate Code logs conversation turns (prompt, tool calls, and assistant response) to improve validation quality and agent behavior. Logs are sent to Altimate's backend and are not shared with third parties.
+
+**To opt out:**
+
+```bash
+export ALTIMATE_LOGGER_DISABLED=true
+```
+
+Add it to your shell profile (`~/.zshrc`, `~/.bashrc`) to make it permanent.
+
+See [`docs/docs/configure/logging.md`](docs/docs/configure/logging.md) for details on what is collected.
+
 ## Community & Contributing
 
 - **Issues**: [GitHub Issues](https://github.com/AltimateAI/altimate-code/issues)
diff --git a/bun.lock b/bun.lock
diff --git a/docs/docs/configure/logging.md b/docs/docs/configure/logging.md
@@ -0,0 +1,60 @@
+# Conversation Logging
+
+Altimate Code automatically logs each conversation turn to the Altimate backend. This powers validation, audit, and quality analysis features. Logging is **enabled by default** — no configuration is required to activate it.
+
+## What Is Logged
+
+Each turn (one user prompt + all assistant responses) sends the following to the Altimate backend:
+
+| Field | Description |
+|-------|-------------|
+| `session_id` | The current session identifier |
+| `conversation_id` | The assistant message ID for this turn |
+| `user_id` | Your email or username (from your Altimate account) |
+| `user_prompt` | The text of your message |
+| `parts` | All reasoning, text, and tool call/response parts from the assistant |
+| `final_response` | The last text response from the assistant |
+| `metadata` | Model ID, token counts, and cost for the turn |
+
+Logging fires after the session becomes idle (i.e., after the assistant finishes responding). Up to 500 messages are captured per turn to ensure complete coverage of multi-step agentic sessions.
+
+## Why We Log
+
+Conversation logs are used to:
+
+- **Validate AI responses** — power the `/validate` skill that audits factual claims against source data
+- **Quality analysis** — identify recurring failure patterns across sessions
+- **Audit trails** — provide a record of what the assistant did and why
+
+## Disabling Logging
+
+Logging is on by default. To disable it, set the following environment variable before starting Altimate Code:
+
+```bash
+export ALTIMATE_LOGGER_DISABLED=true
+```
+
+To make this permanent, add it to your shell profile (`~/.zshrc`, `~/.bashrc`, etc.):
+
+```bash
+echo 'export ALTIMATE_LOGGER_DISABLED=true' >> ~/.zshrc
+source ~/.zshrc
+```
+
+To re-enable logging, unset the variable:
+
+```bash
+unset ALTIMATE_LOGGER_DISABLED
+```
+
+Setting `ALTIMATE_LOGGER_DISABLED=false` is equivalent to not setting it — logging will be active.
+
+## Network
+
+Conversation logs are sent to:
+
+| Endpoint | Purpose |
+|----------|---------|
+| `apimi.tryaltimate.com` | Conversation log ingestion |
+
+Requests are fire-and-forget — a failed log request does not affect your session in any way.
diff --git a/packages/altimate-code/src/provider/models-snapshot.ts b/packages/altimate-code/src/provider/models-snapshot.ts
diff --git a/packages/opencode/package.json b/packages/opencode/package.json
@@ -8,7 +8,7 @@
   "scripts": {
     "typecheck": "tsgo --noEmit",
     "test": "bun test --timeout 30000",
-    "build": "bun run script/build.ts",
+    "build": "bun run script/build.ts && bun run --conditions=browser ./src/index.ts validate install",
     "dev": "bun run --conditions=browser ./src/index.ts",
     "db": "bun drizzle-kit"
   },
diff --git a/packages/opencode/script/build.ts b/packages/opencode/script/build.ts
@@ -71,6 +71,11 @@ const migrations = await Promise.all(
 )
 console.log(`Loaded ${migrations.length} migrations`)
 
+// Load validate skill assets for embedding
+const validateSkillMd = await Bun.file(path.join(dir, "src/skill/validate/SKILL.md")).text()
+const validateBatchPy = await Bun.file(path.join(dir, "src/skill/validate/batch_validate.py")).text()
+console.log("Loaded validate skill assets")
+
 const singleFlag = process.argv.includes("--single")
 const baselineFlag = process.argv.includes("--baseline")
 const skipInstall = process.argv.includes("--skip-install")
@@ -220,6 +225,8 @@ for (const item of targets) {
       OPENCODE_LIBC: item.os === "linux" ? `'${item.abi ?? "glibc"}'` : "undefined",
       OPENCODE_MIGRATIONS: JSON.stringify(migrations),
       OPENCODE_CHANGELOG: JSON.stringify(changelog),
+      ALTIMATE_VALIDATE_SKILL_MD: JSON.stringify(validateSkillMd),
+      ALTIMATE_VALIDATE_BATCH_PY: JSON.stringify(validateBatchPy),
       OTUI_TREE_SITTER_WORKER_PATH: bunfsRoot + workerRelativePath,
     },
   })
diff --git a/packages/opencode/src/cli/cmd/tui/app.tsx b/packages/opencode/src/cli/cmd/tui/app.tsx
@@ -106,6 +106,7 @@ import type { EventSource } from "./context/sdk"
 export function tui(input: {
   url: string
   args: Args
+  config: TuiConfig.Info
   directory?: string
   fetch?: typeof fetch
   headers?: RequestInit["headers"]
diff --git a/packages/opencode/src/cli/cmd/tui/attach.ts b/packages/opencode/src/cli/cmd/tui/attach.ts
@@ -2,6 +2,9 @@ import { cmd } from "../cmd"
 import { UI } from "@/cli/ui"
 import { tui } from "./app"
 import { win32DisableProcessedInput, win32InstallCtrlCGuard } from "./win32"
+import { TuiConfig } from "@/config/tui"
+import { Instance } from "@/project/instance"
+import { existsSync } from "fs"
 
 export const AttachCommand = cmd({
   command: "attach <url>",
@@ -63,8 +66,13 @@ export const AttachCommand = cmd({
         const auth = `Basic ${Buffer.from(`altimate:${password}`).toString("base64")}`
         return { Authorization: auth }
       })()
+      const config = await Instance.provide({
+        directory: directory && existsSync(directory) ? directory : process.cwd(),
+        fn: () => TuiConfig.get(),
+      })
       await tui({
         url: args.url,
+        config,
         args: {
           continue: args.continue,
           sessionID: args.session,
diff --git a/packages/opencode/src/cli/cmd/tui/thread.ts b/packages/opencode/src/cli/cmd/tui/thread.ts
@@ -12,6 +12,8 @@ import { Filesystem } from "@/util/filesystem"
 import type { Event } from "@opencode-ai/sdk/v2"
 import type { EventSource } from "./context/sdk"
 import { win32DisableProcessedInput, win32InstallCtrlCGuard } from "./win32"
+import { TuiConfig } from "@/config/tui"
+import { Instance } from "@/project/instance"
 
 declare global {
   const OPENCODE_WORKER_PATH: string
@@ -135,6 +137,10 @@ export const TuiThreadCommand = cmd({
         if (!args.prompt) return piped
         return piped ? piped + "\n" + args.prompt : args.prompt
       })
+      const config = await Instance.provide({
+        directory: cwd,
+        fn: () => TuiConfig.get(),
+      })
 
       // Check if server should be started (port or hostname explicitly set in CLI or config)
       const networkOpts = await resolveNetworkOptions(args)
@@ -165,6 +171,8 @@ export const TuiThreadCommand = cmd({
 
       const tuiPromise = tui({
         url,
+        config,
+        directory: cwd,
         fetch: customFetch,
         events,
         tuiConfig,
diff --git a/packages/opencode/src/cli/cmd/validate.ts b/packages/opencode/src/cli/cmd/validate.ts
@@ -0,0 +1,83 @@
+import type { Argv } from "yargs"
+import { cmd } from "./cmd"
+import * as prompts from "@clack/prompts"
+import fs from "fs/promises"
+import path from "path"
+import os from "os"
+
+// Injected at build time by build.ts (same pattern as ALTIMATE_CLI_MIGRATIONS).
+// In development these fall back to reading from disk via getAssets().
+declare const ALTIMATE_VALIDATE_SKILL_MD: string
+declare const ALTIMATE_VALIDATE_BATCH_PY: string
+
+interface ValidateAssets {
+  skillMd: string
+  batchPy: string
+}
+
+async function getAssets(): Promise<ValidateAssets> {
+  if (
+    typeof ALTIMATE_VALIDATE_SKILL_MD !== "undefined" &&
+    typeof ALTIMATE_VALIDATE_BATCH_PY !== "undefined"
+  ) {
+    return {
+      skillMd: ALTIMATE_VALIDATE_SKILL_MD,
+      batchPy: ALTIMATE_VALIDATE_BATCH_PY,
+    }
+  }
+  // Development fallback: read from disk relative to this source file
+  const skillsDir = path.join(import.meta.dir, "../../skill/validate")
+  const [skillMd, batchPy] = await Promise.all([
+    fs.readFile(path.join(skillsDir, "SKILL.md"), "utf-8"),
+    fs.readFile(path.join(skillsDir, "batch_validate.py"), "utf-8"),
+  ])
+  return { skillMd, batchPy }
+}
+
+
+
+const InstallSubcommand = cmd({
+  command: "install",
+  describe: "install the /validate skill into ~/.altimate-code",
+  handler: async () => {
+    prompts.intro("Altimate Validate — Installer")
+
+    const { skillMd, batchPy } = await getAssets()
+
+    const spinner = prompts.spinner()
+    spinner.start("Installing /validate skill...")
+    const skillTargetDir = path.join(os.homedir(), ".altimate-code", "skills", "validate")
+    await fs.mkdir(skillTargetDir, { recursive: true })
+    await fs.writeFile(path.join(skillTargetDir, "SKILL.md"), skillMd)
+    await fs.writeFile(path.join(skillTargetDir, "batch_validate.py"), batchPy)
+    spinner.stop(`Installed /validate skill → ${skillTargetDir}`)
+
+    prompts.outro("Altimate validation skill installed successfully!")
+  },
+})
+
+const StatusSubcommand = cmd({
+  command: "status",
+  describe: "check whether the /validate skill is installed",
+  handler: async () => {
+    const skillDir = path.join(os.homedir(), ".altimate-code", "skills", "validate")
+
+    prompts.intro("Altimate Validate — Installation Status")
+
+    const check = (exists: boolean, label: string, detail: string) =>
+      prompts.log.info(`${exists ? "✓" : "✗"} ${label}${exists ? "" : " (not found)"}: ${detail}`)
+
+    const skillMdExists = await fs.access(path.join(skillDir, "SKILL.md")).then(() => true).catch(() => false)
+    const batchPyExists = await fs.access(path.join(skillDir, "batch_validate.py")).then(() => true).catch(() => false)
+    check(skillMdExists && batchPyExists, "/validate skill", skillDir)
+
+    prompts.outro("Done")
+  },
+})
+
+export const ValidateCommand = cmd({
+  command: "validate",
+  describe: "manage the Altimate validation framework (/validate skill)",
+  builder: (yargs: Argv) => yargs.command(InstallSubcommand).command(StatusSubcommand).demandCommand(),
+  handler: () => {},
+})
diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts
@@ -29,6 +29,7 @@ import { WebCommand } from "./cli/cmd/web"
 import { PrCommand } from "./cli/cmd/pr"
 import { SessionCommand } from "./cli/cmd/session"
 import { DbCommand } from "./cli/cmd/db"
+import { ValidateCommand } from "./cli/cmd/validate"
 import path from "path"
 import { Global } from "./global"
 import { JsonMigration } from "./storage/json-migration"
@@ -166,6 +167,7 @@ let cli = yargs(hideBin(process.argv))
   .command(PrCommand)
   .command(SessionCommand)
   .command(DbCommand)
+  .command(ValidateCommand)
 
 if (Installation.isLocal()) {
   cli = cli.command(WorkspaceServeCommand)
diff --git a/packages/opencode/src/installation/index.ts b/packages/opencode/src/installation/index.ts
@@ -177,6 +177,9 @@ export namespace Installation {
         throw new Error(`Unknown method: ${method}`)
     }
     const result = await cmd.quiet().throws(false)
+    // Dynamic import breaks the Installation → Telemetry → Config → ModelsDev → Installation
+    // circular dependency that would cause Installation to be undefined at module load time.
+    const { Telemetry } = await import("../telemetry")
     if (result.exitCode !== 0) {
       const stderr = method === "choco" ? "not running from an elevated command shell" : result.stderr.toString("utf8")
       const telemetryMethod = (["npm", "bun", "brew"].includes(method) ? method : "other") as "npm" | "bun" | "brew" | "other"
diff --git a/packages/opencode/src/project/bootstrap.ts b/packages/opencode/src/project/bootstrap.ts
@@ -12,6 +12,7 @@ import { Log } from "@/util/log"
 import { ShareNext } from "@/share/share-next"
 import { Snapshot } from "../snapshot"
 import { Truncate } from "../tool/truncation"
+import { initConversationLogger } from "../session/conversation-logger"
 
 export async function InstanceBootstrap() {
   Log.Default.info("bootstrapping", { directory: Instance.directory })
@@ -24,6 +25,9 @@ export async function InstanceBootstrap() {
   Vcs.init()
   Snapshot.init()
   Truncate.init()
+  if (process.env.ALTIMATE_LOGGER_DISABLED !== "true") {
+    initConversationLogger()
+  }
 
   Bus.subscribe(Command.Event.Executed, async (payload) => {
     if (payload.properties.name === Command.Default.INIT) {
diff --git a/packages/opencode/src/server/routes/session.ts b/packages/opencode/src/server/routes/session.ts
@@ -19,6 +19,25 @@ import { lazy } from "../../util/lazy"
 
 const log = Log.create({ service: "server" })
 
+function resolvePrompt(sessionID: string, body: Omit<SessionPrompt.PromptInput, "sessionID">) {
+  const textPart = body.parts?.find((p: { type: string }) => p.type === "text")
+  const text = (textPart && "text" in textPart ? (textPart.text as string) : "").trimStart()
+
+  if (text.startsWith("/validate")) {
+    return SessionPrompt.command({
+      sessionID,
+      command: "validate",
+      arguments: text.slice("/validate".length).trim(),
+      model: body.model ? `${body.model.providerID}/${body.model.modelID}` : undefined,
+      agent: body.agent,
+      variant: body.variant,
+      messageID: body.messageID,
+    })
+  }
+
+  return SessionPrompt.prompt({ ...body, sessionID })
+}
+
 export const SessionRoutes = lazy(() =>
   new Hono()
     .get(
@@ -763,7 +782,9 @@ export const SessionRoutes = lazy(() =>
         return stream(c, async (stream) => {
           const sessionID = c.req.valid("param").sessionID
           const body = c.req.valid("json")
-          const msg = await SessionPrompt.prompt({ ...body, sessionID })
+
+          const msg = await resolvePrompt(sessionID, body)
+
           stream.write(JSON.stringify(msg))
         })
       },
@@ -795,7 +816,7 @@ export const SessionRoutes = lazy(() =>
         return stream(c, async () => {
           const sessionID = c.req.valid("param").sessionID
           const body = c.req.valid("json")
-          SessionPrompt.prompt({ ...body, sessionID })
+          resolvePrompt(sessionID, body)
         })
       },
     )
diff --git a/packages/opencode/src/session/conversation-logger.ts b/packages/opencode/src/session/conversation-logger.ts
diff --git a/packages/opencode/src/skill/validate/SKILL.md b/packages/opencode/src/skill/validate/SKILL.md
diff --git a/packages/opencode/src/skill/validate/batch_validate.py b/packages/opencode/src/skill/validate/batch_validate.py
