feat(0.13.0): persistent ExperimentStores — FileSystem + D1 (#12)

* feat(0.13.0): persistent ExperimentStores — FileSystem + D1

Adds two backing stores for ExperimentStore alongside the existing
InMemoryExperimentStore so consumers don't have to roll their own.

FileSystemExperimentStore (Node-only, src/experiment-tracker-fs.ts)
  Mirrors FileSystemTraceStore: append-only NDJSON files for experiments
  + runs under one directory, 32MB rollover, lazy-loaded in-memory index.
  The append log doubles as an audit trail of every state transition.
  Tolerates truncated tail lines from a crash during write.

D1ExperimentStore (Workers-safe, src/experiment-tracker-d1.ts)
  Two-table D1 schema (experiments + runs + meta). Idempotent
  ensureSchema() + INSERT ... ON CONFLICT DO UPDATE so a save-run retry
  never corrupts. Configurable tablePrefix lets multiple stores share
  one D1 binding (tax_eval_runs vs legal_eval_runs). No hard dep on
  @cloudflare/workers-types — exposes a minimal D1Like surface that
  the runtime binding satisfies structurally.

Why now
  Four downstream consumers (tax-agent, legal-agent, gtm-agent,
  film-agent) all surfaced this gap in their AGENT_EVAL_MIGRATION docs:
  agent-eval shipped only InMemoryExperimentStore, so any Worker run
  lost experiment state on restart. This closes the gap upstream so
  the four migration PRs can adopt the canonical primitive instead of
  hand-rolling a fourth different SQL schema.

Also ignores .venv/ and __pycache__/ so the python client's local venv
never leaks into commits again.

Tests: 11 new (FS round-trip, NDJSON layout, audit history, rollover,
list ordering, scoped listRuns, crash tolerance + D1 round-trip,
scoped listRuns, tablePrefix isolation, update-not-append on retry).
Suite: 552 → 563 passing.

* chore: ignore python venv + bytecode caches

The python client at clients/python keeps its venv inside the repo;
without these patterns the venv blob leaks into commits.

Author: drewstone

.gitignore

@@ -2,3 +2,9 @@ node_modules/
 dist/
 .env
 *.tsbuildinfo
+
+# Python clients (venvs + bytecode caches should never enter git)
+.venv/
+**/__pycache__/
+*.pyc
+*.pyo

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Trace-first evaluation framework for Tangle agents. Core (spans, pipelines, sandbox harness, OTLP export), trust (dataset, red-team, calibration, behavior DSL), builder-of-builders (three-layer eval, resumable sessions, meta-runtime correlation), and frontier (meta-eval correlation study, Process Reward Modeling, bisector).",
   "type": "module",
   "main": "./dist/index.js",

src/experiment-tracker-d1.ts

@@ -0,0 +1,245 @@
+/**
+ * D1ExperimentStore — Cloudflare D1-backed `ExperimentStore`.
+ *
+ * Workers-safe (uses only the `D1Database` binding the runtime injects). Two
+ * tables, no joins, no migrations beyond `ensureSchema()`. Schema designed so
+ * a Worker route can both write the row at run start and update it at run end
+ * without losing the original config — the row's lifecycle mirrors the
+ * `Run.status` field one-to-one.
+ *
+ * Why this lives next to `InMemoryExperimentStore`:
+ *   - bad-app, legal-agent, gtm-agent, film-agent all run as Workers
+ *   - Workers cannot use `node:fs`, so `FileSystemExperimentStore` doesn't apply
+ *   - Hand-rolling D1 SQL in every consumer is exactly the duplication this
+ *     module exists to prevent
+ *
+ * Schema versioning: the `meta` table records `schema_version` so a future
+ * column addition can be detected and migrated additively. Today's schema is
+ * v1; bump only on breaking shape changes.
+ */
+
+import type { Experiment, ExperimentStore, Run } from './experiment-tracker'
+
+/**
+ * Minimal `D1Database` shape we depend on. Avoids pulling in
+ * `@cloudflare/workers-types` as a hard dep — consumers that already have
+ * those types installed can pass the binding directly.
+ */
+export interface D1Like {
+  prepare(query: string): D1PreparedStatementLike
+  batch?(statements: D1PreparedStatementLike[]): Promise<unknown[]>
+  exec(query: string): Promise<unknown>
+}
+
+export interface D1PreparedStatementLike {
+  bind(...values: unknown[]): D1PreparedStatementLike
+  first<T = Record<string, unknown>>(): Promise<T | null>
+  all<T = Record<string, unknown>>(): Promise<{ results: T[] }>
+  run(): Promise<unknown>
+}
+
+export interface D1ExperimentStoreOptions {
+  /** D1 binding from `env`. */
+  db: D1Like
+  /**
+   * Optional table-name prefix so multiple ExperimentStores can share a DB
+   * without colliding (e.g. `tax_eval_experiments` vs `legal_eval_experiments`).
+   * Default: `agent_eval_`.
+   */
+  tablePrefix?: string
+}
+
+const SCHEMA_VERSION = 1
+
+export class D1ExperimentStore implements ExperimentStore {
+  private readonly db: D1Like
+  private readonly experimentsTable: string
+  private readonly runsTable: string
+  private readonly metaTable: string
+  private schemaReady = false
+
+  constructor(options: D1ExperimentStoreOptions) {
+    this.db = options.db
+    const prefix = options.tablePrefix ?? 'agent_eval_'
+    this.experimentsTable = `${prefix}experiments`
+    this.runsTable = `${prefix}runs`
+    this.metaTable = `${prefix}meta`
+  }
+
+  /**
+   * Idempotent schema setup. Safe to call before every operation; the second
+   * call short-circuits via `schemaReady`. Most consumers will call it once
+   * during Worker bootstrap.
+   */
+  async ensureSchema(): Promise<void> {
+    if (this.schemaReady) return
+    // Single `exec` so D1 batches the DDL.
+    const ddl = `
+      CREATE TABLE IF NOT EXISTS ${this.experimentsTable} (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        metadata_json TEXT
+      );
+      CREATE TABLE IF NOT EXISTS ${this.runsTable} (
+        id TEXT PRIMARY KEY,
+        experiment_id TEXT NOT NULL,
+        name TEXT,
+        status TEXT NOT NULL,
+        started_at TEXT NOT NULL,
+        completed_at TEXT,
+        config_json TEXT NOT NULL,
+        report_json TEXT,
+        error TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_${this.runsTable}_experiment ON ${this.runsTable}(experiment_id);
+      CREATE INDEX IF NOT EXISTS idx_${this.runsTable}_started ON ${this.runsTable}(started_at);
+      CREATE TABLE IF NOT EXISTS ${this.metaTable} (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      INSERT OR REPLACE INTO ${this.metaTable}(key, value) VALUES ('schema_version', '${SCHEMA_VERSION}');
+    `
+    await this.db.exec(ddl.trim().replace(/\s+/g, ' '))
+    this.schemaReady = true
+  }
+
+  async saveExperiment(exp: Experiment): Promise<void> {
+    await this.ensureSchema()
+    await this.db
+      .prepare(
+        `INSERT INTO ${this.experimentsTable}(id, name, created_at, metadata_json)
+         VALUES (?1, ?2, ?3, ?4)
+         ON CONFLICT(id) DO UPDATE SET
+           name = excluded.name,
+           created_at = excluded.created_at,
+           metadata_json = excluded.metadata_json`,
+      )
+      .bind(exp.id, exp.name, exp.createdAt, exp.metadata ? JSON.stringify(exp.metadata) : null)
+      .run()
+  }
+
+  async getExperiment(id: string): Promise<Experiment | null> {
+    await this.ensureSchema()
+    const row = await this.db
+      .prepare(
+        `SELECT id, name, created_at, metadata_json
+         FROM ${this.experimentsTable}
+         WHERE id = ?1`,
+      )
+      .bind(id)
+      .first<ExperimentRow>()
+    return row ? rowToExperiment(row) : null
+  }
+
+  async listExperiments(): Promise<Experiment[]> {
+    await this.ensureSchema()
+    const { results } = await this.db
+      .prepare(
+        `SELECT id, name, created_at, metadata_json
+         FROM ${this.experimentsTable}
+         ORDER BY created_at DESC`,
+      )
+      .all<ExperimentRow>()
+    return results.map(rowToExperiment)
+  }
+
+  async saveRun(run: Run): Promise<void> {
+    await this.ensureSchema()
+    await this.db
+      .prepare(
+        `INSERT INTO ${this.runsTable}(id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error)
+         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
+         ON CONFLICT(id) DO UPDATE SET
+           experiment_id = excluded.experiment_id,
+           name = excluded.name,
+           status = excluded.status,
+           started_at = excluded.started_at,
+           completed_at = excluded.completed_at,
+           config_json = excluded.config_json,
+           report_json = excluded.report_json,
+           error = excluded.error`,
+      )
+      .bind(
+        run.id,
+        run.experimentId,
+        run.name ?? null,
+        run.status,
+        run.startedAt,
+        run.completedAt ?? null,
+        JSON.stringify(run.config),
+        run.report ? JSON.stringify(run.report) : null,
+        run.error ?? null,
+      )
+      .run()
+  }
+
+  async getRun(id: string): Promise<Run | null> {
+    await this.ensureSchema()
+    const row = await this.db
+      .prepare(
+        `SELECT id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error
+         FROM ${this.runsTable}
+         WHERE id = ?1`,
+      )
+      .bind(id)
+      .first<RunRow>()
+    return row ? rowToRun(row) : null
+  }
+
+  async listRuns(experimentId: string): Promise<Run[]> {
+    await this.ensureSchema()
+    const { results } = await this.db
+      .prepare(
+        `SELECT id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error
+         FROM ${this.runsTable}
+         WHERE experiment_id = ?1
+         ORDER BY started_at DESC`,
+      )
+      .bind(experimentId)
+      .all<RunRow>()
+    return results.map(rowToRun)
+  }
+}
+
+interface ExperimentRow {
+  id: string
+  name: string
+  created_at: string
+  metadata_json: string | null
+}
+
+interface RunRow {
+  id: string
+  experiment_id: string
+  name: string | null
+  status: string
+  started_at: string
+  completed_at: string | null
+  config_json: string
+  report_json: string | null
+  error: string | null
+}
+
+function rowToExperiment(row: ExperimentRow): Experiment {
+  return {
+    id: row.id,
+    name: row.name,
+    createdAt: row.created_at,
+    ...(row.metadata_json ? { metadata: JSON.parse(row.metadata_json) as Record<string, unknown> } : {}),
+  }
+}
+
+function rowToRun(row: RunRow): Run {
+  return {
+    id: row.id,
+    experimentId: row.experiment_id,
+    ...(row.name ? { name: row.name } : {}),
+    status: row.status as Run['status'],
+    startedAt: row.started_at,
+    ...(row.completed_at ? { completedAt: row.completed_at } : {}),
+    config: JSON.parse(row.config_json),
+    ...(row.report_json ? { report: JSON.parse(row.report_json) } : {}),
+    ...(row.error ? { error: row.error } : {}),
+  }
+}