render_results.ts

// README results-table generator (render_results.ts)
// ===================================================
//
// TypeScript port of render_results.py. Reads the latest eval CSVs and renders the
// "Quality & Evaluation" Markdown table, then injects it into README.md between:
//
//     <!-- EVAL:START -->
//     ... generated table ...
//     <!-- EVAL:END -->
//
// If the markers are absent, the table is printed to stdout with a hint. This keeps the
// published results table reproducible from the committed CSV artifacts.
//
// Run: pnpm --filter @readmycareer/eval render            # inject into README
//      pnpm --filter @readmycareer/eval render -- --print # print only

import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { readFileSync, writeFileSync, existsSync } from "node:fs";
import { readRows } from "./lib/csv";
import { hasFlag } from "./lib/args";

const HERE = dirname(fileURLToPath(import.meta.url));
const EVAL_DIR = resolve(HERE, "..");
const REPO_ROOT = resolve(HERE, "..", "..");
const README = resolve(REPO_ROOT, "README.md");
const START = "<!-- EVAL:START -->";
const END = "<!-- EVAL:END -->";

function toBool(v: string | undefined): boolean | null {
  if (v === undefined || v === "") return null;
  return ["true", "1", "yes"].includes(v.trim().toLowerCase());
}

function toFloat(v: string | undefined): number | null {
  if (v === undefined || v === "") return null;
  const n = Number(v);
  // Reject NaN and non-finite (Inf) so they never leak into means/rates.
  return Number.isFinite(n) ? n : null;
}

function mean(values: number[]): number {
  return values.length ? values.reduce((a, b) => a + b, 0) / values.length : NaN;
}

function rate(flags: (boolean | null)[]): number {
  const present = flags.filter((f): f is boolean => f !== null);
  return present.length ? present.filter((f) => f).length / present.length : NaN;
}

function p95(values: number[]): number {
  if (values.length === 0) return NaN;
  const s = [...values].sort((a, b) => a - b);
  const idx = Math.max(0, Math.min(Math.trunc(s.length * 0.95), s.length - 1));
  return s[idx];
}

function pct(v: number): string {
  return Number.isNaN(v) ? "N/A" : `${(v * 100).toFixed(1)}%`;
}

function num(v: number, digits: number): string {
  return Number.isNaN(v) ? "N/A" : v.toFixed(digits);
}

type Row = [string, string, string]; // metric, value, threshold

function agentMetrics(): Row[] {
  const rows = readRows(resolve(EVAL_DIR, "agent_harness_results.csv"));
  if (rows.length === 0) return [];
  const floats = (col: string): number[] =>
    rows.map((r) => toFloat(r[col])).filter((f): f is number => f !== null);
  return [
    ["Schema Compliance", pct(rate(rows.map((r) => toBool(r["schema_valid"])))), "≥ 95%"],
    ["Gap Recall (vs labels)", pct(mean(floats("gap_recall"))), "≥ 50%"],
    ["Gap Faithfulness (LLM-judge)", num(mean(floats("gap_faithfulness")), 3), "≥ 0.70"],
    ["Plan Completeness", pct(rate(rows.map((r) => toBool(r["plan_completeness"])))), "≥ 90%"],
    ["Date Consistency", pct(rate(rows.map((r) => toBool(r["date_consistent"])))), "= 100%"],
    ["p95 Latency", num(p95(floats("latency_s")), 1) + "s", "< 30s"],
    ["Avg Cost / Request", "$" + num(mean(floats("cost_usd")), 4), "< $0.01"],
  ];
}

function ragMetrics(): Row[] {
  const out: Row[] = [];
  const rag = readRows(resolve(EVAL_DIR, "ragas_results.csv"));
  const cols: [string, string][] = [
    ["faithfulness", "RAG Faithfulness"],
    ["answer_relevancy", "Answer Relevancy"],
    ["context_precision", "Context Precision"],
    ["context_recall", "Context Recall"],
  ];
  for (const [col, label] of cols) {
    const vals = rag.map((r) => toFloat(r[col])).filter((f): f is number => f !== null);
    if (vals.length) out.push([label, num(mean(vals), 3), "≥ 0.70"]);
  }

  const grounding = readRows(resolve(EVAL_DIR, "grounding_results.csv"));
  const targeted = grounding.filter((r) => r["expected_doc_type"]);
  if (targeted.length) {
    out.push([
      "Grounding / Citation Rate",
      pct(rate(targeted.map((r) => toBool(r["grounded"])))),
      "≥ 80%",
    ]);
  }
  return out;
}

function todayIso(): string {
  const d = new Date();
  const pad = (n: number) => String(n).padStart(2, "0");
  return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
}

function buildMarkdown(): string {
  const agent = agentMetrics();
  const rag = ragMetrics();
  const lines: string[] = [];
  lines.push(
    `_Generated by \`eval/render_results.ts\` on ${todayIso()} from the committed eval CSVs._`
  );
  lines.push("");

  if (agent.length) {
    lines.push("**Agent Output Quality**");
    lines.push("");
    lines.push("| Metric | Score | Threshold |");
    lines.push("| --- | --- | --- |");
    for (const [metric, value, threshold] of agent) {
      lines.push(`| ${metric} | ${value} | ${threshold} |`);
    }
    lines.push("");
  }

  if (rag.length) {
    lines.push("**RAG Retrieval Quality**");
    lines.push("");
    lines.push("| Metric | Score | Threshold |");
    lines.push("| --- | --- | --- |");
    for (const [metric, value, threshold] of rag) {
      lines.push(`| ${metric} | ${value} | ${threshold} |`);
    }
    lines.push("");
  }

  if (!agent.length && !rag.length) {
    lines.push("_No eval results found. Run `pnpm eval` to populate the CSVs._");
    lines.push("");
  }

  return lines.join("\n").replace(/\s+$/, "") + "\n";
}

function inject(markdown: string): boolean {
  if (!existsSync(README)) {
    console.log("README.md not found.");
    return false;
  }
  const text = readFileSync(README, "utf8");
  if (!text.includes(START) || !text.includes(END)) {
    console.log(`Markers ${START} / ${END} not found in README.md — printing table instead:\n`);
    console.log(markdown);
    console.log(`\nAdd the markers to README.md where the table should appear:\n${START}\n${END}`);
    return false;
  }
  const pre = text.split(START)[0];
  const post = text.split(END).slice(1).join(END);
  writeFileSync(README, `${pre}${START}\n${markdown}${END}${post}`, "utf8");
  console.log(`Injected results table into ${README}`);
  return true;
}

export function main(argv: string[]): number {
  const markdown = buildMarkdown();
  if (hasFlag(argv, "--print")) {
    console.log(markdown);
    return 0;
  }
  inject(markdown);
  return 0;
}

const INVOKED_DIRECTLY = process.argv[1] === fileURLToPath(import.meta.url);
if (INVOKED_DIRECTLY) {
  process.exit(main(process.argv.slice(2)));
}