Add built-in HTML reporter (#36)

Author: CAMOBAP

docs/reporters.md

@@ -11,12 +11,13 @@ skillgym run <suite.ts> --reporter standard
 skillgym run <suite.ts> --reporter json
 skillgym run <suite.ts> --reporter json-summary
 skillgym run <suite.ts> --reporter github-actions
+skillgym run <suite.ts> --reporter html
 skillgym run <suite.ts> --reporter ./examples/custom-reporter.ts
 skillgym run <suite.ts> --schedule isolated-by-runner --max-parallel 4
 ```
 
 - Omitting `--reporter` uses the built-in `standard` reporter.
-- Built-in reporters are `standard`, `json`, `json-summary`, and `github-actions`.
+- Built-in reporters are `standard`, `json`, `json-summary`, `github-actions`, and `html`.
 - Relative paths resolve from `process.cwd()`.
 
 ## Config
@@ -137,6 +138,17 @@ The built-in `json-summary` reporter writes a trimmed JSON summary to stdout —
 - Per-runner results include `failureClass` when present so downstream tooling can keep grouped-failure semantics.
 - It is useful for post-run analysis steps or feeding results to an LLM.
 
+## HTML reporter
+
+The built-in `html` reporter writes a self-contained `report.html` file to the suite run artifact directory.
+
+- It renders a collapsible tree: suite summary → case → runner execution → individual session events.
+- Each runner block shows pass/fail status, duration, and token usage (input/output).
+- Assistant messages, tool calls, file reads, and commands are rendered as expandable `<details>` elements.
+- The final output of each execution is shown in a collapsible block.
+- Errors are shown inline when present.
+- The output path is printed to stdout after the file is written.
+
 ## GitHub Actions reporter
 
 The built-in `github-actions` reporter is designed for GitHub CI.

src/index.ts

@@ -45,6 +45,7 @@ export { assert, CommandMatcherBuilder, commandMatcher } from "./assertions/inde
 export {
   BUILT_IN_REPORTER_NAMES,
   createGitHubActionsReporter,
+  createHtmlReporter,
   createJsonReporter,
   createStandardReporter,
   loadReporter,

src/reporters/builtins.ts

@@ -3,6 +3,7 @@ export const BUILT_IN_REPORTER_NAMES = [
   "json",
   "json-summary",
   "github-actions",
+  "html",
 ] as const;
 
 export type BuiltInReporterName = (typeof BUILT_IN_REPORTER_NAMES)[number];

src/reporters/html.ts

@@ -0,0 +1,133 @@
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { CaseResult, RunnerResult, SuiteRunResult } from "../domain/result.js";
+import type { Case } from "../domain/case.js";
+import type { SessionEvent } from "../domain/session-report.js";
+import type { BenchmarkReporter, CaseFinishEvent } from "./contract.js";
+
+function esc(str: string): string {
+  return str
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}
+
+function ms(n: number | undefined): string {
+  if (n == null) return "—";
+  return n < 1000 ? `${n}ms` : `${(n / 1000).toFixed(1)}s`;
+}
+
+function tokens(n: number | undefined): string {
+  if (n == null) return "—";
+  return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n);
+}
+
+function renderEvent(e: SessionEvent): string {
+  if (e.type === "message" && e.role === "assistant") {
+    return `<details><summary>assistant message</summary><pre>${esc(e.text)}</pre></details>`;
+  }
+  if (e.type === "toolCall") {
+    const args = e.args ? JSON.stringify(e.args, null, 2) : "";
+    return `<details><summary>tool: ${esc(e.tool)}</summary><pre>${esc(args)}</pre></details>`;
+  }
+  if (e.type === "fileRead") {
+    return `<p>read: <code>${esc(e.path)}</code></p>`;
+  }
+  if (e.type === "command") {
+    return `<p>cmd: <code>${esc(e.command)}</code></p>`;
+  }
+  return "";
+}
+
+function renderRunnerBlock(rr: RunnerResult): string {
+  const report = rr.report;
+  const statusLabel = rr.passed ? "PASS" : "FAIL";
+  const runnerId = rr.runner?.id ?? "unknown";
+  const eventsHtml = (report?.events ?? []).map(renderEvent).join("\n");
+  const errorHtml = rr.error
+    ? `<pre>${esc(rr.error.message)}\n${esc(rr.error.stack ?? "")}</pre>`
+    : "";
+  const finalOutput = report?.finalOutput
+    ? `<details><summary>final output</summary><pre>${esc(report.finalOutput)}</pre></details>`
+    : "";
+
+  return `
+    <section>
+      <h3>${esc(runnerId)} — ${statusLabel} — ${ms(rr.durationMs)}
+        &nbsp; in: ${tokens(report?.usage?.inputTokens)} / out: ${tokens(report?.usage?.outputTokens)}
+      </h3>
+      ${errorHtml}
+      ${finalOutput}
+      ${eventsHtml}
+    </section>`;
+}
+
+function renderCaseSection(testCase: Case, caseResult: CaseResult): string {
+  const status = caseResult.passed ? "✓" : "✗";
+  const runnerBlocks = (caseResult.runnerResults ?? [])
+    .filter(Boolean)
+    .map(renderRunnerBlock)
+    .join("\n");
+
+  return `
+    <details>
+      <summary><strong>${status} ${esc(testCase.id)}</strong></summary>
+      <blockquote><pre>${esc(testCase.prompt)}</pre></blockquote>
+      ${runnerBlocks}
+    </details>`;
+}
+
+function renderPage(result: SuiteRunResult, caseSections: string): string {
+  const passed = result.cases.filter((c) => c.passed).length;
+  const total = result.cases.length;
+
+  return `
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+      <meta charset="UTF-8">
+      <title>skillgym — ${esc(result.suitePath)}</title>
+      <style>
+        body { font-family: monospace; margin: 2em; }
+        pre { white-space: pre-wrap; word-break: break-word; background: #f4f4f4; padding: 0.5em; }
+        summary { cursor: pointer; }
+        blockquote { border-left: 3px solid #ccc; margin: 0; padding-left: 1em; }
+        h3 { margin: 0.5em 0; font-size: 1em; }
+        section { margin-left: 1em; border-left: 2px solid #eee; padding-left: 1em; }
+      </style>
+    </head>
+    <body>
+      <h1>skillgym report</h1>
+      <p>${esc(result.suitePath)}</p>
+      <p>${passed}/${total} passed &nbsp; ${ms(result.durationMs)}</p>
+      <hr>
+      ${caseSections}
+    </body>
+    </html>`;
+}
+
+export function createHtmlReporter(): BenchmarkReporter {
+  const caseResults: CaseFinishEvent[] = [];
+
+  return {
+    onCaseFinish(event) {
+      caseResults.push(event);
+    },
+
+    onSuiteFinish(event) {
+      try {
+        const { result } = event;
+        const caseSections = caseResults
+          .map((ev) => renderCaseSection(ev.case, ev.result))
+          .join("\n");
+        const html = renderPage(result, caseSections);
+        const outPath = join(result.suiteRunArtifactDir, "report.html");
+        writeFileSync(outPath, html, "utf-8");
+        console.log(`\nHTML report: ${outPath}\n`);
+      } catch (err) {
+        console.error("html-reporter error:", err);
+      }
+    },
+  };
+}

src/reporters/index.ts

@@ -11,6 +11,7 @@ export type {
 } from "./contract.js";
 export { BUILT_IN_REPORTER_NAMES } from "./builtins.js";
 export { createGitHubActionsReporter } from "./github-actions.js";
+export { createHtmlReporter } from "./html.js";
 export { createJsonReporter } from "./json.js";
 export { createJsonSummaryReporter } from "./json-summary.js";
 export { loadReporter } from "./load-reporter.js";

src/reporters/load-reporter.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import type { BenchmarkReporter } from "./contract.js";
 import { createGitHubActionsReporter } from "./github-actions.js";
+import { createHtmlReporter } from "./html.js";
 import { createJsonReporter } from "./json.js";
 import { createJsonSummaryReporter } from "./json-summary.js";
 import { createStandardReporter } from "./standard.js";
@@ -40,6 +41,8 @@ export async function loadReporter(
         return createJsonSummaryReporter();
       case "github-actions":
         return createGitHubActionsReporter();
+      case "html":
+        return createHtmlReporter();
     }
   }