docs(benchmark): annotate benchmark corpus and runners

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

Author: tae2089

internal/benchmark/analyze.go

@@ -1,10 +1,12 @@
+// @index Match analysis helpers for benchmark run results.
 package benchmark
 
 import "strings"
 
 // fileMatches reports whether actual matches expected, supporting both
 // exact match and suffix match to handle absolute vs relative path differences.
 // e.g. expected "internal/webhook/handler.go" matches actual "/home/user/repo/internal/webhook/handler.go"
+// @intent compare expected files against recorded reads without depending on absolute path stability.
 func fileMatches(expected, actual string) bool {
 	if expected == actual {
 		return true
@@ -15,6 +17,7 @@ func fileMatches(expected, actual string) bool {
 // MatchFiles computes the ratio of expected files found in FilesRead or mentioned
 // in the Answer text (as a fallback when tool-call data is unavailable).
 // Returns 1.0 if no expected files are specified.
+// @intent score whether a benchmark run inspected the files the query was expected to touch.
 func MatchFiles(result RunResult, query Query) float64 {
 	if len(query.ExpectedFiles) == 0 {
 		return 1.0
@@ -41,6 +44,7 @@ func MatchFiles(result RunResult, query Query) float64 {
 
 // MatchSymbols computes the ratio of expected symbols found in the answer text.
 // Returns 1.0 if no expected symbols are specified.
+// @intent score whether the final answer mentioned the symbols the query was expected to surface.
 func MatchSymbols(result RunResult, query Query) float64 {
 	if len(query.ExpectedSymbols) == 0 {
 		return 1.0
@@ -55,6 +59,7 @@ func MatchSymbols(result RunResult, query Query) float64 {
 }
 
 // CountCcgToolCalls returns the number of tool calls using mcp__ccg__ prefix.
+// @intent quantify how much a benchmark run relied on CCG-specific MCP tools.
 func CountCcgToolCalls(result RunResult) int {
 	var count int
 	for _, tc := range result.ToolCalls {
@@ -66,6 +71,7 @@ func CountCcgToolCalls(result RunResult) int {
 }
 
 // ComputeMatch derives a MatchResult from a single RunResult and its Query.
+// @intent consolidate per-query benchmark scoring into one reusable result structure.
 func ComputeMatch(result RunResult, query Query) MatchResult {
 	return MatchResult{
 		QueryID:          result.QueryID,
@@ -79,6 +85,7 @@ func ComputeMatch(result RunResult, query Query) MatchResult {
 
 // AnalyzeRun computes MatchResult for every result in the run against the corpus.
 // Results with no matching query are skipped.
+// @intent turn a full benchmark run into per-query scored matches against the corpus definition.
 func AnalyzeRun(run *BenchmarkRun, corpus *Corpus) []MatchResult {
 	queryMap := make(map[string]Query, len(corpus.Queries))
 	for _, q := range corpus.Queries {

internal/benchmark/compare.go

@@ -1,7 +1,9 @@
+// @index Comparison helpers for benchmark runs with and without CCG.
 package benchmark
 
 // Compare builds a ComparisonReport from a with-ccg run and an optional without-ccg run.
 // Matches contains with-ccg metrics; MatchesWithout contains without-ccg metrics when provided.
+// @intent produce one report that captures both benchmark modes and their scored query matches.
 func Compare(withCCG *BenchmarkRun, withoutCCG *BenchmarkRun, corpus *Corpus) *ComparisonReport {
 	report := &ComparisonReport{
 		WithCCG:    withCCG,

internal/benchmark/corpus.go

@@ -1,3 +1,4 @@
+// @index Benchmark corpus loading, saving, and validation utilities.
 package benchmark
 
 import (
@@ -12,6 +13,7 @@ import (
 var validQueryID = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
 
 // LoadCorpus reads a queries.yaml file and validates its contents.
+// @intent load a benchmark corpus from disk while enforcing schema and ID constraints up front.
 func LoadCorpus(path string) (*Corpus, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -28,6 +30,7 @@ func LoadCorpus(path string) (*Corpus, error) {
 }
 
 // SaveCorpus writes a Corpus to a YAML file.
+// @intent persist benchmark corpus definitions in the YAML format used by the CLI.
 func SaveCorpus(path string, c *Corpus) error {
 	data, err := yaml.Marshal(c)
 	if err != nil {
@@ -40,6 +43,7 @@ func SaveCorpus(path string, c *Corpus) error {
 }
 
 // ValidateCorpus checks that all queries have required fields and no duplicate IDs.
+// @intent reject malformed benchmark corpora before any benchmark execution depends on them.
 func ValidateCorpus(c *Corpus) error {
 	if len(c.Queries) == 0 {
 		return fmt.Errorf("corpus has no queries")

internal/benchmark/jsonl.go

@@ -1,3 +1,4 @@
+// @index JSONL session parsing and extraction helpers for benchmark analysis.
 package benchmark
 
 import (
@@ -15,6 +16,7 @@ var (
 )
 
 // SessionMessage represents one line from a Claude Code session JSONL file.
+// @intent model the subset of Claude session JSONL needed to reconstruct benchmark runs.
 type SessionMessage struct {
 	Type      string          `json:"type"`
 	Message   *MessagePayload `json:"message,omitempty"`
@@ -23,13 +25,15 @@ type SessionMessage struct {
 }
 
 // MessagePayload is the nested message object inside a SessionMessage.
+// @intent expose assistant role, content blocks, and usage data from session lines.
 type MessagePayload struct {
 	Role    string          `json:"role"`
 	Content []ContentBlock  `json:"content"`
 	Usage   *UsageInfo      `json:"usage,omitempty"`
 }
 
 // ContentBlock is a single content item (text, tool_use, etc.).
+// @intent represent individual Claude message blocks so benchmark analyzers can inspect tool calls and text.
 type ContentBlock struct {
 	Type  string          `json:"type"`
 	Text  string          `json:"text,omitempty"`
@@ -39,19 +43,22 @@ type ContentBlock struct {
 }
 
 // UsageInfo holds token usage from Claude's response.
+// @intent capture token accounting for one Claude message so benchmark runs can aggregate usage.
 type UsageInfo struct {
 	InputTokens  int `json:"input_tokens"`
 	OutputTokens int `json:"output_tokens"`
 }
 
 // QuerySegment groups the messages belonging to a single benchmark query.
+// @intent isolate the portion of a session that belongs to one marked benchmark query.
 type QuerySegment struct {
 	QueryID  string
 	Messages []SessionMessage
 }
 
 // ParseJSONL reads a Claude Code session JSONL file and returns all parsed lines.
 // Lines that are not valid JSON are silently skipped.
+// @intent ingest recorded Claude sessions even when they contain partial or non-JSON noise lines.
 func ParseJSONL(path string) ([]SessionMessage, error) {
 	f, err := os.Open(path)
 	if err != nil {
@@ -83,6 +90,7 @@ const (
 
 // extractMarker checks if a SessionMessage contains a benchmark marker.
 // Returns (queryID, isStart, found). Uses strict regex to prevent injection via crafted IDs.
+// @intent locate benchmark query boundaries inside session transcripts safely.
 func extractMarker(m SessionMessage) (queryID string, isStart bool, found bool) {
 	var text string
 	if m.Type == "tool_result" {
@@ -107,6 +115,7 @@ func extractMarker(m SessionMessage) (queryID string, isStart bool, found bool)
 
 // ExtractQuerySegments splits a session's messages into per-query segments using markers.
 // An unclosed segment (START without END) is included, spanning to the last message.
+// @intent reconstruct per-query transcript slices from a whole benchmark session log.
 func ExtractQuerySegments(msgs []SessionMessage) ([]QuerySegment, error) {
 	var segs []QuerySegment
 	var current *QuerySegment
@@ -134,6 +143,7 @@ func ExtractQuerySegments(msgs []SessionMessage) ([]QuerySegment, error) {
 }
 
 // ExtractToolCalls collects all tool_use blocks from a segment's messages.
+// @intent recover structured tool invocation history for one benchmark query.
 func ExtractToolCalls(seg QuerySegment) []ToolCall {
 	var calls []ToolCall
 	for _, m := range seg.Messages {
@@ -154,6 +164,7 @@ func ExtractToolCalls(seg QuerySegment) []ToolCall {
 }
 
 // ExtractFilesRead returns deduplicated file paths from Read tool calls in the segment.
+// @intent estimate file-inspection behavior from tool-use records without double-counting reads.
 func ExtractFilesRead(seg QuerySegment) []string {
 	var files []string
 	seen := make(map[string]bool)
@@ -178,6 +189,7 @@ func ExtractFilesRead(seg QuerySegment) []string {
 }
 
 // ExtractTokens sums input and output token counts across all messages in a segment.
+// @intent aggregate token usage for the full lifecycle of one benchmark query.
 func ExtractTokens(seg QuerySegment) (inputTokens, outputTokens int) {
 	for _, m := range seg.Messages {
 		if m.Message != nil && m.Message.Usage != nil {
@@ -189,6 +201,7 @@ func ExtractTokens(seg QuerySegment) (inputTokens, outputTokens int) {
 }
 
 // ExtractAnswer returns the text of the last assistant text block in the segment.
+// @intent capture the final assistant answer that should be evaluated for symbol hits.
 func ExtractAnswer(seg QuerySegment) string {
 	var last string
 	for _, m := range seg.Messages {
@@ -205,6 +218,7 @@ func ExtractAnswer(seg QuerySegment) string {
 }
 
 // ExtractRunResult builds a RunResult from a query segment.
+// @intent convert an extracted query segment into the canonical benchmark run result shape.
 func ExtractRunResult(queryID string, seg QuerySegment) RunResult {
 	in, out := ExtractTokens(seg)
 	return RunResult{

internal/benchmark/report.go

@@ -1,3 +1,4 @@
+// @index Markdown report generation for benchmark comparisons.
 package benchmark
 
 import (
@@ -7,6 +8,7 @@ import (
 )
 
 // WriteReport renders a ComparisonReport as markdown and writes it to outPath.
+// @intent turn benchmark comparison data into a readable artifact for sharing and regression tracking.
 func WriteReport(report *ComparisonReport, outPath string) error {
 	if report.WithCCG == nil {
 		return fmt.Errorf("comparison report missing with-ccg run")

internal/benchmark/runner.go

@@ -1,3 +1,4 @@
+// @index Benchmark runner orchestration for Claude CLI subprocess execution.
 package benchmark
 
 import (
@@ -11,11 +12,13 @@ import (
 
 // Executor abstracts the subprocess execution so tests can inject a mock.
 // dir sets the working directory for the subprocess; empty means inherit from parent.
+// @intent decouple benchmark orchestration from the concrete Claude CLI process implementation.
 type Executor interface {
 	Execute(ctx context.Context, args []string, prompt, dir string) ([]byte, error)
 }
 
 // RunnerConfig holds configuration for a benchmark run.
+// @intent collect execution mode, workspace, and timeout settings for one benchmark run.
 type RunnerConfig struct {
 	Mode         string // "with-ccg" | "without-ccg"
 	CWD          string // benchmark workspace directory
@@ -24,6 +27,7 @@ type RunnerConfig struct {
 }
 
 // DefaultRunnerConfig returns a RunnerConfig with sensible defaults.
+// @intent provide conservative benchmark defaults that work for most local runs.
 func DefaultRunnerConfig() RunnerConfig {
 	return RunnerConfig{
 		MaxToolCalls: 50,
@@ -32,6 +36,7 @@ func DefaultRunnerConfig() RunnerConfig {
 }
 
 // claudeOutput is the structure of `claude -p --output-format json` output.
+// @intent decode the subset of Claude CLI JSON output needed for benchmark scoring.
 type claudeOutput struct {
 	IsError bool   `json:"is_error"`
 	Result  string `json:"result"`
@@ -43,6 +48,7 @@ type claudeOutput struct {
 
 // buildArgs constructs CLI args for a specific run.
 // mcpConfigFile is the path to an empty MCP config file for without-ccg mode.
+// @intent centralize Claude CLI argument construction for both benchmark modes.
 func buildArgs(cfg RunnerConfig, mcpConfigFile string) []string {
 	args := []string{"-p", "--output-format", "json"}
 	if cfg.MaxToolCalls > 0 {
@@ -58,6 +64,7 @@ func buildArgs(cfg RunnerConfig, mcpConfigFile string) []string {
 // Working directory is set on the subprocess directly (not via --cwd flag).
 // In "without-ccg" mode, --strict-mcp-config disables all MCP servers.
 // NOTE: For actual runs, use Runner.Run which creates a real temp file for --mcp-config.
+// @intent expose deterministic argument construction for tests and tooling that inspect benchmark invocation behavior.
 func BuildClaudeArgs(cfg RunnerConfig) []string {
 	if cfg.Mode == "without-ccg" {
 		return buildArgs(cfg, `{"mcpServers":{}}`)
@@ -67,6 +74,7 @@ func BuildClaudeArgs(cfg RunnerConfig) []string {
 
 // BuildPrompt creates the prompt string that wraps a query with benchmark markers.
 // The markers allow the JSONL analyzer to locate query boundaries.
+// @intent surround benchmark queries with markers so later session analysis can recover exact query segments.
 func BuildPrompt(q Query) string {
 	return fmt.Sprintf(
 		"===BENCHMARK_QUERY_START id=%s===\n%s\n===BENCHMARK_QUERY_END id=%s===",
@@ -75,18 +83,21 @@ func BuildPrompt(q Query) string {
 }
 
 // Runner executes benchmark queries sequentially using the provided Executor.
+// @intent own the end-to-end lifecycle of benchmark query execution for one mode.
 type Runner struct {
 	cfg  RunnerConfig
 	exec Executor
 }
 
 // NewRunner creates a Runner with the given config and executor.
+// @intent assemble benchmark orchestration from execution config and a subprocess adapter.
 func NewRunner(cfg RunnerConfig, exec Executor) *Runner {
 	return &Runner{cfg: cfg, exec: exec}
 }
 
 // Run executes each query in the corpus and returns a BenchmarkRun.
 // Executor errors are captured in RunResult.Error rather than aborting the run.
+// @intent execute the full benchmark corpus while preserving per-query failures in the final report.
 func (r *Runner) Run(ctx context.Context, corpus *Corpus) (*BenchmarkRun, error) {
 	run := &BenchmarkRun{
 		Mode:    r.cfg.Mode,
@@ -136,6 +147,7 @@ func (r *Runner) Run(ctx context.Context, corpus *Corpus) (*BenchmarkRun, error)
 // prepareMCPConfig creates a temp JSON file for without-ccg mode so that
 // --mcp-config receives a file path rather than inline JSON (avoiding the
 // variadic flag consuming the prompt from stdin).
+// @intent disable MCP cleanly in without-ccg mode without confusing the Claude CLI argument parser.
 func (r *Runner) prepareMCPConfig() (path string, cleanup func(), err error) {
 	if r.cfg.Mode != "without-ccg" {
 		return "", func() {}, nil
@@ -154,6 +166,7 @@ func (r *Runner) prepareMCPConfig() (path string, cleanup func(), err error) {
 }
 
 // parseClaudeOutput extracts token counts and answer text from claude's JSON output.
+// @intent translate Claude CLI JSON output into benchmark result fields without failing the whole run on parse misses.
 func (r *Runner) parseClaudeOutput(out []byte, res *RunResult) {
 	if len(out) == 0 {
 		return

internal/benchmark/schema.go

@@ -4,6 +4,7 @@ package benchmark
 import "time"
 
 // Query represents a single benchmark query with expected results.
+// @intent define one benchmark prompt and the files or symbols it is expected to surface.
 type Query struct {
 	ID              string   `yaml:"id"               json:"id"`
 	Description     string   `yaml:"description"      json:"description"`
@@ -13,18 +14,21 @@ type Query struct {
 }
 
 // Corpus holds the collection of benchmark queries.
+// @intent group benchmark queries into a reusable corpus that can be run and validated together.
 type Corpus struct {
 	Version string  `yaml:"version" json:"version,omitempty"`
 	Queries []Query `yaml:"queries" json:"queries"`
 }
 
 // ToolCall records a single tool invocation during query execution.
+// @intent capture the tool usage footprint of one benchmarked query execution.
 type ToolCall struct {
 	Tool  string `json:"tool"`
 	Input string `json:"input,omitempty"`
 }
 
 // RunResult captures the outcome of executing a single query.
+// @intent store answer text, tool usage, timing, and token counts for one benchmark query.
 type RunResult struct {
 	QueryID      string     `json:"query_id"`
 	ToolCalls    []ToolCall `json:"tool_calls,omitempty"`
@@ -37,13 +41,15 @@ type RunResult struct {
 }
 
 // BenchmarkRun holds all results from a single benchmark execution.
+// @intent represent one complete benchmark session for a given execution mode.
 type BenchmarkRun struct {
 	Mode    string      `json:"mode"`
 	RunAt   time.Time   `json:"run_at"`
 	Results []RunResult `json:"results"`
 }
 
 // ResultByID returns the RunResult for the given query ID, or nil if not found.
+// @intent support direct lookup of a query result inside a benchmark run.
 func (r *BenchmarkRun) ResultByID(id string) *RunResult {
 	for i := range r.Results {
 		if r.Results[i].QueryID == id {
@@ -54,6 +60,7 @@ func (r *BenchmarkRun) ResultByID(id string) *RunResult {
 }
 
 // MatchResult holds the computed match metrics for a single query.
+// @intent summarize scored file, symbol, tool, and token metrics for one benchmark query.
 type MatchResult struct {
 	QueryID          string  `json:"query_id"`
 	FileHitRatio     float64 `json:"file_hit_ratio"`
@@ -64,6 +71,7 @@ type MatchResult struct {
 }
 
 // ComparisonReport holds a comparison between two benchmark runs.
+// @intent package benchmark runs and their scored matches into one comparison artifact.
 type ComparisonReport struct {
 	WithCCG        *BenchmarkRun `json:"with_ccg"`
 	WithoutCCG     *BenchmarkRun `json:"without_ccg,omitempty"`