From ee8b1046812f159aaac1a40e8fb0e237c53a8b4d Mon Sep 17 00:00:00 2001 From: Raushan Singh Date: Mon, 4 May 2026 14:49:18 +0530 Subject: [PATCH 1/6] feat: add self-improving benchmark pipeline for OAPE tools Add a benchmark pipeline that measures and improves the quality of the OAPE code generation tools (api-generate, api-implement) using a feedback loop against real EP implementations. The pipeline: 1. Takes curated EP-to-implementation mappings (EP URL + repo + PRs) 2. Clones the operator repo at the pre-EP commit (bias-free) 3. Runs the OAPE tools to generate code from the EP 4. Compares output against the real merged implementation 5. An improver agent analyzes gaps and edits the tool instructions 6. Repeats with the improved tool (default: 3 iterations) 7. Restores original tool files and reports score progression Key features: - Bias prevention: repo cloned before EP merge, agent never sees truth - Multi-PR support: combines related PRs into single ground truth - File classification: distinguishes auto-generated artifacts, formatting changes, valuable extras, and genuinely wrong files - Adjusted precision: doesn't penalize the tool for generating tests or sample configs the human didn't write - Generic improvements: tool edits are pattern-based, not EP-specific - Optional PR push: promote any iteration's output to a real PR - All agents use configurable model (default: claude-opus-4-6 max) New files: - benchmark/benchmark.py - CLI entry point and orchestrator - benchmark/runner.py - Generation + improvement feedback loop - benchmark/compare.py - Delta-aware diff, scoring, classification - benchmark/isolate.py - PR timeline resolution, bias-free env - benchmark/ground_truth.py - Combined truth extraction from PRs - benchmark/report.py - Markdown + JSON report generation - benchmark/models.py - Shared data models - benchmark/go_ast_helper/ - Go AST extraction for struct comparison - benchmark/config.yaml - Example config with EP #1863 - benchmark/README.md - Full usage documentation Co-authored-by: Cursor --- README.md | 17 + benchmark/.gitignore | 3 + benchmark/README.md | 241 ++++++++++++++ benchmark/benchmark.py | 378 ++++++++++++++++++++++ benchmark/compare.py | 532 +++++++++++++++++++++++++++++++ benchmark/config.yaml | 15 + benchmark/go_ast_helper/go.mod | 3 + benchmark/go_ast_helper/main.go | 219 +++++++++++++ benchmark/ground_truth.py | 96 ++++++ benchmark/isolate.py | 126 ++++++++ benchmark/models.py | 136 ++++++++ benchmark/report.py | 353 +++++++++++++++++++++ benchmark/requirements.txt | 3 + benchmark/runner.py | 545 ++++++++++++++++++++++++++++++++ 14 files changed, 2667 insertions(+) create mode 100644 benchmark/.gitignore create mode 100644 benchmark/README.md create mode 100644 benchmark/benchmark.py create mode 100644 benchmark/compare.py create mode 100644 benchmark/config.yaml create mode 100644 benchmark/go_ast_helper/go.mod create mode 100644 benchmark/go_ast_helper/main.go create mode 100644 benchmark/ground_truth.py create mode 100644 benchmark/isolate.py create mode 100644 benchmark/models.py create mode 100644 benchmark/report.py create mode 100644 benchmark/requirements.txt create mode 100644 benchmark/runner.py diff --git a/README.md b/README.md index 758c3e6..f7e89bc 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,23 @@ Automatically applies code fixes from a review report. /oape:e2e-generate main ``` +## Benchmark Pipeline + +The project includes a self-improving benchmark pipeline that measures and improves the quality of the OAPE code generation tools. It works by: + +1. Taking EPs that have already been implemented by humans +2. Re-generating the implementation using the OAPE tools (in a bias-free environment) +3. Comparing the output against the real merged code +4. Iteratively improving the tool instructions based on the gaps found + +```bash +# Run the benchmark feedback loop +cd benchmark/ +python benchmark.py run --config config.yaml +``` + +See [benchmark/README.md](benchmark/README.md) for full documentation, including how to add your own EPs for benchmarking. + ## Deployment ### Build the container images diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 0000000..9738538 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,3 @@ +results/ +__pycache__/ +*.pyc diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..e65e075 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,241 @@ +# OAPE Benchmark Pipeline + +A self-improving benchmark pipeline for the OAPE code generation tools. It takes Enhancement Proposals (EPs) that have already been implemented by humans, re-generates the implementation using the OAPE tools, compares the output against the real merged code, and iteratively improves the tool instructions based on the gaps found. + +## How It Works + +``` +EP #1863 + Repo + PR numbers + │ + ▼ +┌──────────────────────────────────────────────────┐ +│ Iteration 1: Generate with ORIGINAL tool │ +│ ├── Clone repo at pre-EP commit (bias-free) │ +│ ├── Run /oape:api-generate + /oape:api-implement│ +│ ├── Compare output vs human implementation │ +│ └── Score: completeness, precision, conventions │ +├──────────────────────────────────────────────────┤ +│ Improve: Analyze gaps, edit tool instructions │ +├──────────────────────────────────────────────────┤ +│ Iteration 2: Generate with IMPROVED-v1 tool │ +│ ├── Fresh clone at same pre-EP commit │ +│ ├── Run tools with improved instructions │ +│ └── Compare and score again │ +├──────────────────────────────────────────────────┤ +│ Improve: Further refine tool instructions │ +├──────────────────────────────────────────────────┤ +│ Iteration 3: Generate with IMPROVED-v2 tool │ +│ └── Final comparison and report │ +└──────────────────────────────────────────────────┘ + │ + ▼ + Report: score progression, tool diffs, file classification +``` + +### Bias Prevention + +The generating agent never sees the existing implementation. The repo is cloned at the commit **before** the EP was merged, so the EP's code physically does not exist. The agent only receives the EP URL and generates from scratch. + +### Scoring + +- **Completeness**: What % of the human's structs/fields/functions did the tool also produce? +- **Raw Precision**: What % of generated code matches the human's output? (penalizes all extras) +- **Adjusted Precision**: Same as raw but excludes auto-generated artifacts (`zz_generated`, CRDs, bundle manifests) and valuable extras (tests, sample configs) +- **Convention Compliance**: Kubebuilder marker and naming pattern match rate +- **Build Success**: Does `make build` pass? + +Extra files are classified into four categories: +- **Auto-generated**: Output of `make generate`/`make manifests` (not errors) +- **Formatting-only**: Whitespace/import reordering (not errors) +- **Valuable extras**: Tests, validation, sample configs the human didn't write (tool outperformed human) +- **Genuinely wrong**: Files that should not have been touched (real errors) + +## Prerequisites + +- Python 3.11+ +- `gh` CLI authenticated (`gh auth login`) +- `git` +- `go` toolchain +- `claude-agent-sdk` (`pip install claude-agent-sdk`) +- Claude Code CLI with access to `claude-opus-4-6` (or another model) + +## Quick Start + +### 1. Create a config file + +Create `benchmark/config.yaml` with your EP-to-implementation mappings: + +```yaml +benchmark_cases: + - ep_url: "https://github.com/openshift/enhancements/pull/1863" + repo_url: "https://github.com/openshift/zero-trust-workload-identity-manager" + description: "SPIRE federation support" + implementation_prs: [68, 82] + + # Add more EPs here: + # - ep_url: "https://github.com/openshift/enhancements/pull/XXXX" + # repo_url: "https://github.com/openshift/" + # description: "Short description" + # implementation_prs: [PR1, PR2] # all PRs that implement this EP + +settings: + model: "claude-opus-4-6" # Claude model to use + effort: "max" # Effort level (low/medium/high/max) + iterations: 3 # Number of feedback loop iterations + output_dir: "benchmark/results" + parallel: false +``` + +**How to find the right values:** + +| Field | How to find it | +|-------|---------------| +| `ep_url` | The Enhancement Proposal PR on `openshift/enhancements` | +| `repo_url` | The upstream `openshift/` repo where the EP was implemented | +| `implementation_prs` | The PR numbers **on the operator repo** (not the EP repo) that implement this EP. Use `gh pr list --repo openshift/ --state merged --search ""` to find them | +| `description` | Short label for reports | + +**Important**: The EP URL and implementation PRs are on **different repos**. The EP is on `openshift/enhancements`, the PRs are on `openshift/`. + +### 2. Run the benchmark + +```bash +cd benchmark/ + +# Run the feedback loop (iteratively improves the tool) +python benchmark.py run --config config.yaml + +# Force re-run if results already exist +python benchmark.py run --config config.yaml --force + +# Override number of iterations +python benchmark.py run --config config.yaml --iterations 5 + +# Ad-hoc single EP (no config file needed) +python benchmark.py run \ + --ep-url "https://github.com/openshift/enhancements/pull/1863" \ + --repo "https://github.com/openshift/zero-trust-workload-identity-manager" \ + --prs 68,82 +``` + +### 3. Review results + +Results are written to `benchmark/results//ep-/`: + +``` +results/ + zero-trust-workload-identity-manager/ + ep-1863/ + report.md # Human-readable report with score progression + report.json # Machine-readable scores + truth/ # Ground truth from human implementation + combined.diff + files_added.txt + files_modified.txt + iter-1/ # Iteration 1 output + diff.patch # What the tool generated + scores.json + iter-2/ # Iteration 2 (with improved-v1 tool) + diff.patch + scores.json + iter-3/ # Iteration 3 (with improved-v2 tool) + diff.patch + scores.json + tool-backups/ # Tool instruction snapshots + original/ # Original api-generate.md, api-implement.md + after-improvement-1/ + after-improvement-2/ + tool-improvement-after-iter-1/ # Exact diffs of tool changes + tool-improvement-after-iter-2/ +``` + +### 4. Generate aggregate report (when running multiple EPs) + +```bash +python benchmark.py report --results-dir benchmark/results/ +``` + +### 5. Push a specific iteration as a PR (optional) + +If an iteration produced good output, you can push it as a PR: + +```bash +python benchmark.py push \ + --ep 1863 \ + --repo "https://github.com//" \ + --iteration 3 \ + --base-branch main \ + --title "feat: implement EP #1863 (OAPE generated)" +``` + +## Adding Your Own EPs + +To benchmark against your team's EPs: + +1. **Find EPs with merged implementations**: You need EPs where the code is already merged on the upstream `openshift/` operator repo. The benchmark compares tool output against the merged implementation. + +2. **Identify the implementation PRs**: For each EP, find which PRs on the operator repo implement it. There's often no direct link between the EP and the code PRs (different Jira tickets, no EP URL in PR body), so you need to know which PRs are related. + + ```bash + # Search for related PRs + gh pr list --repo openshift/ --state merged \ + --search "federation OR SPIRE-54" \ + --json number,title,mergedAt --limit 20 + ``` + +3. **Add to config.yaml**: Add a new entry with the EP URL, repo URL, and all related PR numbers. + +4. **Run**: The pipeline handles everything else -- cloning at the right commit, running the tools, comparing, scoring, and improving. + +### Multi-PR EPs + +When an EP is implemented across multiple PRs (common for large features), list all PR numbers: + +```yaml +- ep_url: "https://github.com/openshift/enhancements/pull/1863" + repo_url: "https://github.com/openshift/zero-trust-workload-identity-manager" + description: "SPIRE federation support" + implementation_prs: [68, 82] # PR #68 + PR #82 combined as ground truth +``` + +The pipeline automatically: +- Sorts PRs by merge date +- Uses the parent of the earliest PR as the baseline (pre-EP state) +- Combines all PR diffs as the ground truth +- Handles file overlaps between PRs (uses final state) + +## Architecture + +``` +benchmark.py CLI entry point, orchestrates the pipeline +isolate.py Clones repo at pre-EP commit, verifies no bias +ground_truth.py Extracts combined diff from all implementation PRs +runner.py Runs OAPE tools + improver agent via Claude Agent SDK +compare.py Delta-aware diff, scoring, file classification +report.py Generates markdown + JSON reports +models.py Shared data models +go_ast_helper/ Go program for AST-level struct/field comparison +config.yaml Benchmark case definitions (user-provided) +``` + +### Tool Improvement Loop + +The improver agent (a separate Claude invocation) receives: +- The comparison results (scores, missed files, wrong structs) +- The ground truth diff and generated diff +- The current tool instruction files (`api-generate.md`, `api-implement.md`) + +It makes **generic** improvements to the instructions -- not EP-specific ones. This means improvements from benchmarking one EP benefit all future EP implementations. + +After the benchmark completes, original tool files are **automatically restored**. Improved versions are preserved in `tool-backups/` for you to review and selectively adopt. + +## Cost Estimates + +| Component | Approximate cost | +|-----------|-----------------| +| Generation (per iteration) | $5-15 | +| Tool improvement (per iteration) | $2-5 | +| Full 3-iteration run | $25-35 | +| 10 EPs x 3 iterations | $250-350 | + +Costs vary by EP complexity and model. Using `effort: "max"` is more expensive but produces better results. diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py new file mode 100644 index 0000000..dc87655 --- /dev/null +++ b/benchmark/benchmark.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""OAPE Benchmark Pipeline CLI. + +Runs OAPE tools against curated EP-to-implementation mappings, +compares generated output vs real merged code, and reports quality. +""" + +import argparse +import asyncio +import json +import logging +import shutil +import subprocess +import sys +from pathlib import Path + +import yaml + +from compare import compare_all_iterations +from ground_truth import extract_combined_truth +from isolate import prepare_generation_env, resolve_pr_timeline +from models import BenchmarkCase, BenchmarkConfig, BenchmarkResult +from report import generate_aggregate_report, generate_ep_report +from runner import run_feedback_loop, PLUGIN_DIR + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%H:%M:%S", +) +logger = logging.getLogger("benchmark") + + +def load_config(config_path: str, iterations_override: int | None = None) -> BenchmarkConfig: + with open(config_path) as f: + raw = yaml.safe_load(f) + + cases = [] + for entry in raw.get("benchmark_cases", []): + cases.append(BenchmarkCase( + ep_url=entry["ep_url"], + repo_url=entry["repo_url"], + description=entry.get("description", ""), + implementation_prs=entry["implementation_prs"], + )) + + settings = raw.get("settings", {}) + iterations = iterations_override or settings.get("iterations", 3) + + return BenchmarkConfig( + cases=cases, + tools_to_benchmark=settings.get("tools_to_benchmark", ["api-generate", "api-implement"]), + iterations=iterations, + output_dir=settings.get("output_dir", "benchmark/results"), + parallel=settings.get("parallel", False), + model=settings.get("model", "claude-opus-4-6"), + effort=settings.get("effort", "max"), + ) + + +async def run_single_case( + case: BenchmarkCase, + config: BenchmarkConfig, + output_dir: Path, + force: bool = False, +) -> BenchmarkResult | None: + """Run the benchmark pipeline for a single EP case.""" + ep_num = case.ep_url.rstrip("/").split("/")[-1] + repo_name = case.repo_url.rstrip("/").split("/")[-1].removesuffix(".git") + result_dir = output_dir / repo_name / f"ep-{ep_num}" + + if result_dir.exists() and not force: + report_json = result_dir / "report.json" + if report_json.exists(): + logger.info("Skipping EP #%s (already completed; use --force to re-run)", ep_num) + return None + + logger.info("=" * 60) + logger.info("FEEDBACK LOOP BENCHMARK: EP #%s - %s", ep_num, case.description) + logger.info("Repo: %s", case.repo_url) + logger.info("Implementation PRs: %s", case.implementation_prs) + logger.info("Iterations: %d (with tool improvement between each)", config.iterations) + logger.info("Model: %s | Effort: %s", config.model, config.effort) + logger.info("=" * 60) + + logger.info("Step 1: Resolving PR timeline...") + timeline = resolve_pr_timeline(case.repo_url, case.implementation_prs) + logger.info( + "Timeline: baseline=%s, final=%s, %d files changed", + timeline.earliest_parent_sha[:12], + timeline.latest_merge_sha[:12], + len(timeline.all_changed_files), + ) + + logger.info("Step 2: Extracting ground truth...") + truth = extract_combined_truth(case.repo_url, timeline) + logger.info( + "Ground truth: %d added, %d modified files", + len(truth.files_added), len(truth.files_modified), + ) + + logger.info("Step 3: Preparing isolated generation environment...") + source_env = prepare_generation_env(case.repo_url, timeline) + if source_env.warnings: + for w in source_env.warnings: + logger.warning(w) + + backup_dir = output_dir / repo_name / f"ep-{ep_num}" / "tool-backups" + backup_dir.mkdir(parents=True, exist_ok=True) + + logger.info("Step 4: Running feedback loop (%d iterations)...", config.iterations) + gen_results, improvements = await run_feedback_loop( + ep_url=case.ep_url, + source_env=source_env, + truth=truth, + num_iterations=config.iterations, + plugin_dir=PLUGIN_DIR, + backup_dir=backup_dir, + model=config.model, + effort=config.effort, + ) + + logger.info("Step 5: Final comparison across all iterations...") + benchmark_result = compare_all_iterations( + gen_results=gen_results, + truth=truth, + ep_url=case.ep_url, + repo_url=case.repo_url, + description=case.description, + implementation_prs=case.implementation_prs, + ) + benchmark_result.tool_improvements = improvements + benchmark_result.mode = "feedback_loop" + + logger.info("Step 6: Generating report...") + report_path = generate_ep_report( + result=benchmark_result, + gen_results=gen_results, + truth=truth, + output_dir=output_dir, + ) + + logger.info("Report: %s", report_path) + logger.info("=" * 40) + logger.info("SCORE PROGRESSION:") + for s in benchmark_result.iteration_scores: + tool_ver = gen_results[s.iteration - 1].tool_version if s.iteration <= len(gen_results) else "?" + logger.info( + " Iter %d (%s): completeness=%.1f%% precision=%.1f%% build=%s", + s.iteration, tool_ver, s.completeness, s.precision, s.build_success, + ) + total_improve_cost = sum(imp.improvement_cost_usd for imp in improvements) + total_gen_cost = sum(g.cost_usd for g in gen_results) + logger.info("Total cost: generation=$%.2f + improvement=$%.2f = $%.2f", + total_gen_cost, total_improve_cost, total_gen_cost + total_improve_cost) + logger.info("=" * 40) + + if benchmark_result.outperformance_findings: + logger.info( + "Tool outperformed human in %d cases", + len(benchmark_result.outperformance_findings), + ) + + shutil.rmtree(source_env.path, ignore_errors=True) + + return benchmark_result + + +async def cmd_run(args: argparse.Namespace) -> None: + config = load_config(args.config, args.iterations) + output_dir = Path(config.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + results: list[BenchmarkResult] = [] + + for case in config.cases: + result = await run_single_case(case, config, output_dir, force=args.force) + if result: + results.append(result) + + if results: + logger.info("\n" + "=" * 60) + logger.info("ALL BENCHMARKS COMPLETE") + for r in results: + ep_num = r.ep_url.rstrip("/").split("/")[-1] + logger.info( + " EP #%s: completeness=%.1f%% precision=%.1f%%", + ep_num, r.median_completeness, r.median_precision, + ) + logger.info("=" * 60) + + if len(results) > 1: + agg_path = generate_aggregate_report(results, output_dir) + logger.info("Aggregate report: %s", agg_path) + + +async def cmd_report(args: argparse.Namespace) -> None: + results_dir = Path(args.results_dir) + if not results_dir.exists(): + logger.error("Results directory not found: %s", results_dir) + sys.exit(1) + + results: list[BenchmarkResult] = [] + for report_json in results_dir.rglob("report.json"): + with open(report_json) as f: + data = json.load(f) + from models import IterationScore + scores = [ + IterationScore( + iteration=s["iteration"], + completeness=s["completeness"], + precision=s["precision"], + convention_compliance=s["convention_compliance"], + build_success=s["build_success"], + file_true_positives=s.get("file_tp", 0), + file_false_negatives=s.get("file_fn", 0), + file_false_positives=s.get("file_fp", 0), + ) + for s in data.get("iteration_scores", []) + ] + results.append(BenchmarkResult( + ep_url=data["ep_url"], + repo_url=data["repo_url"], + description=data.get("description", ""), + implementation_prs=data.get("implementation_prs", []), + iteration_scores=scores, + median_completeness=data.get("median_completeness", 0), + median_precision=data.get("median_precision", 0), + best_iteration=data.get("best_iteration", 0), + score_variance=data.get("score_variance", {}), + )) + + if not results: + logger.error("No benchmark results found in %s", results_dir) + sys.exit(1) + + agg_path = generate_aggregate_report(results, results_dir) + logger.info("Aggregate report: %s", agg_path) + + +def cmd_push(args: argparse.Namespace) -> None: + results_dir = Path(args.results_dir or "benchmark/results") + repo_name = args.repo.rstrip("/").split("/")[-1].removesuffix(".git") + iter_dir = results_dir / repo_name / f"ep-{args.ep}" / f"iter-{args.iteration}" + + if not iter_dir.exists(): + logger.error("Iteration directory not found: %s", iter_dir) + sys.exit(1) + + patch_file = iter_dir / "diff.patch" + if not patch_file.exists() or patch_file.stat().st_size == 0: + logger.error("No diff.patch found or patch is empty in %s", iter_dir) + sys.exit(1) + + gen_dir = iter_dir / "generated" + + import tempfile + work_dir = Path(tempfile.mkdtemp(prefix="oape-bench-push-")) + + logger.info("Cloning %s for PR creation...", args.repo) + subprocess.run( + ["git", "clone", "--quiet", args.repo, str(work_dir)], + check=True, + ) + subprocess.run( + ["git", "checkout", "-b", f"oape-benchmark/ep-{args.ep}-iter{args.iteration}"], + cwd=work_dir, check=True, + ) + + logger.info("Applying patch from iteration %d...", args.iteration) + patch_content = patch_file.read_text() + result = subprocess.run( + ["git", "apply", "--verbose", "-"], + input=patch_content, text=True, + cwd=work_dir, capture_output=True, + ) + if result.returncode != 0: + logger.error("Failed to apply patch: %s", result.stderr) + sys.exit(1) + + subprocess.run(["git", "add", "-A"], cwd=work_dir, check=True) + subprocess.run( + ["git", "commit", "-m", args.title or f"feat: OAPE benchmark EP-{args.ep} iteration {args.iteration}"], + cwd=work_dir, check=True, + ) + + logger.info("Pushing branch...") + subprocess.run( + ["git", "push", "-u", "origin", f"oape-benchmark/ep-{args.ep}-iter{args.iteration}"], + cwd=work_dir, check=True, + ) + + logger.info("Creating PR...") + pr_result = subprocess.run( + ["gh", "pr", "create", + "--repo", args.repo, + "--base", args.base_branch, + "--title", args.title or f"feat: OAPE benchmark EP-{args.ep} iteration {args.iteration}", + "--body", f"Generated by OAPE benchmark pipeline (EP #{args.ep}, iteration {args.iteration})"], + cwd=work_dir, capture_output=True, text=True, + ) + if pr_result.returncode == 0: + logger.info("PR created: %s", pr_result.stdout.strip()) + else: + logger.error("Failed to create PR: %s", pr_result.stderr) + + shutil.rmtree(work_dir, ignore_errors=True) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="OAPE Benchmark Pipeline", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + run_parser = subparsers.add_parser("run", help="Run benchmark cases") + run_parser.add_argument("--config", default="benchmark/config.yaml", help="Config YAML file") + run_parser.add_argument("--iterations", type=int, help="Override iteration count") + run_parser.add_argument("--force", action="store_true", help="Re-run completed cases") + run_parser.add_argument("--ep-url", help="Single EP URL (ad-hoc mode)") + run_parser.add_argument("--repo", help="Repo URL (ad-hoc mode)") + run_parser.add_argument("--prs", help="Comma-separated PR numbers (ad-hoc mode)") + + report_parser = subparsers.add_parser("report", help="Generate aggregate report") + report_parser.add_argument("--results-dir", default="benchmark/results", help="Results directory") + + full_parser = subparsers.add_parser("full", help="Run benchmarks + generate aggregate report") + full_parser.add_argument("--config", default="benchmark/config.yaml", help="Config YAML file") + full_parser.add_argument("--iterations", type=int, help="Override iteration count") + full_parser.add_argument("--force", action="store_true", help="Re-run completed cases") + + push_parser = subparsers.add_parser("push", help="Push an iteration as a PR") + push_parser.add_argument("--ep", required=True, help="EP number") + push_parser.add_argument("--repo", required=True, help="Repo URL") + push_parser.add_argument("--iteration", type=int, required=True, help="Iteration number") + push_parser.add_argument("--base-branch", default="main", help="Base branch for PR") + push_parser.add_argument("--title", help="PR title") + push_parser.add_argument("--results-dir", help="Results directory") + + args = parser.parse_args() + + if args.command == "run": + if args.ep_url and args.repo and args.prs: + prs = [int(p.strip()) for p in args.prs.split(",")] + config_dict = { + "benchmark_cases": [{ + "ep_url": args.ep_url, + "repo_url": args.repo, + "description": "Ad-hoc benchmark", + "implementation_prs": prs, + }], + "settings": { + "iterations": args.iterations or 3, + "output_dir": "benchmark/results", + }, + } + import tempfile + tmp = Path(tempfile.mktemp(suffix=".yaml")) + tmp.write_text(yaml.dump(config_dict)) + args.config = str(tmp) + + asyncio.run(cmd_run(args)) + + elif args.command == "report": + asyncio.run(cmd_report(args)) + + elif args.command == "full": + asyncio.run(cmd_run(args)) + args_report = argparse.Namespace(results_dir=load_config(args.config).output_dir) + asyncio.run(cmd_report(args_report)) + + elif args.command == "push": + cmd_push(args) + + +if __name__ == "__main__": + main() diff --git a/benchmark/compare.py b/benchmark/compare.py new file mode 100644 index 0000000..7e1187b --- /dev/null +++ b/benchmark/compare.py @@ -0,0 +1,532 @@ +"""Delta-aware diff engine and scoring for benchmark comparisons. + +Compares generated output against ground truth at file-level and +Go AST-level. Detects cases where the tool outperformed the human. +""" + +import json +import logging +import re +import statistics +import subprocess +from pathlib import Path + +from models import ( + BenchmarkResult, + FileClassification, + GenerationResult, + GroundTruth, + IterationScore, + OutperformanceFinding, +) + +logger = logging.getLogger(__name__) + +MARKER_RE = re.compile(r'//\s*\+kubebuilder:\S+') +STRUCT_RE = re.compile(r'type\s+(\w+)\s+struct\s*\{') +FIELD_RE = re.compile( + r'\s+(\w+)\s+(\S+.*?)\s+`(.*?)`' +) +FUNC_RE = re.compile(r'func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\(') +CONST_RE = re.compile(r'\s+(\w+)\s+\w+\s*=\s*"([^"]*)"') + +AUTO_GENERATED_PATTERNS = [ + "zz_generated", + "bundle/manifests/", + "bundle/metadata/", + "config/crd/bases/", + "config/crd/patches/", + "bundle.Dockerfile", + "config/samples/kustomization.yaml", +] + +FORMATTING_ONLY_EXTENSIONS = [ + ".yaml", ".yml", +] + +VALIDATION_MARKERS = [ + "+kubebuilder:validation:Required", + "+kubebuilder:validation:Optional", + "+kubebuilder:validation:Enum", + "+kubebuilder:validation:Pattern", + "+kubebuilder:validation:Minimum", + "+kubebuilder:validation:Maximum", + "+kubebuilder:validation:MinItems", + "+kubebuilder:validation:MaxItems", + "+kubebuilder:validation:XValidation", + "+kubebuilder:default", + "+optional", +] + + +def _extract_go_elements(content: str) -> dict: + """Extract Go code elements from file content using regex.""" + structs: list[dict] = [] + functions: list[str] = [] + markers: list[str] = [] + consts: list[str] = [] + + for match in STRUCT_RE.finditer(content): + structs.append({"name": match.group(1), "fields": []}) + + for match in FIELD_RE.finditer(content): + field_info = { + "name": match.group(1), + "type": match.group(2).strip(), + "tags": match.group(3), + } + if structs: + structs[-1]["fields"].append(field_info) + + for match in FUNC_RE.finditer(content): + functions.append(match.group(1)) + + for match in MARKER_RE.finditer(content): + markers.append(match.group(0).strip().lstrip("// ")) + + for match in CONST_RE.finditer(content): + consts.append(match.group(1)) + + return { + "structs": structs, + "functions": functions, + "markers": markers, + "consts": consts, + } + + +def _extract_added_lines(diff_text: str) -> list[str]: + """Extract only added lines (starting with +) from a unified diff.""" + added = [] + for line in diff_text.split("\n"): + if line.startswith("+") and not line.startswith("+++"): + added.append(line[1:]) + return added + + +def _try_go_ast_helper(filepath: str) -> dict | None: + """Try to use the Go AST helper for more accurate extraction.""" + helper = Path(__file__).parent / "go_ast_helper" / "main.go" + if not helper.exists(): + return None + try: + result = subprocess.run( + ["go", "run", str(helper), "extract", "--file", filepath], + capture_output=True, text=True, timeout=30, + ) + if result.returncode == 0: + return json.loads(result.stdout) + except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError): + pass + return None + + +def _is_auto_generated(filepath: str) -> bool: + """Check if a file is auto-generated by make/build commands.""" + for pattern in AUTO_GENERATED_PATTERNS: + if pattern in filepath: + return True + return False + + +def _is_formatting_only_change(gen_path: Path, baseline_path: Path | None) -> bool: + """Check if changes to a file are only whitespace/formatting.""" + if baseline_path is None or not baseline_path.exists(): + return False + if not gen_path.exists(): + return False + try: + result = subprocess.run( + ["diff", "--ignore-all-space", "--ignore-blank-lines", + str(baseline_path), str(gen_path)], + capture_output=True, text=True, + ) + return result.returncode == 0 + except FileNotFoundError: + return False + + +def _classify_extra_files( + extra_files: set[str], + gen_result: "GenerationResult", + truth: "GroundTruth", +) -> FileClassification: + """Classify files the tool generated that aren't in the ground truth.""" + classification = FileClassification() + + for fpath in sorted(extra_files): + if _is_auto_generated(fpath): + classification.auto_generated.append(fpath) + continue + + gen_path = gen_result.gen_dir / fpath + if not gen_path.exists(): + classification.genuinely_wrong.append(fpath) + continue + + if fpath.endswith(".go"): + content = gen_path.read_text(errors="replace") + elements = _extract_go_elements(content) + + has_relevant_code = bool( + elements.get("structs") + or elements.get("functions") + or elements.get("consts") + ) + + if has_relevant_code: + has_tests = "_test.go" in fpath + has_validation = any("validation" in m.lower() for m in elements.get("markers", [])) + has_new_types = bool(elements.get("structs")) + + if has_tests or has_validation or has_new_types: + classification.valuable_extra.append(fpath) + else: + classification.genuinely_wrong.append(fpath) + else: + classification.formatting_only.append(fpath) + elif any(fpath.endswith(ext) for ext in FORMATTING_ONLY_EXTENSIONS): + if _is_auto_generated(fpath): + classification.auto_generated.append(fpath) + else: + classification.valuable_extra.append(fpath) + else: + classification.genuinely_wrong.append(fpath) + + return classification + + +def _compute_file_metrics( + gen_files: set[str], + truth_files: set[str], +) -> tuple[int, int, int]: + """Compute file-level true positives, false negatives, false positives.""" + tp = len(gen_files & truth_files) + fn = len(truth_files - gen_files) + fp = len(gen_files - truth_files) + return tp, fn, fp + + +def _compute_element_overlap(gen_elements: dict, truth_elements: dict) -> dict: + """Compute overlap between generated and truth Go elements.""" + metrics = {} + + gen_struct_names = {s["name"] for s in gen_elements.get("structs", [])} + truth_struct_names = {s["name"] for s in truth_elements.get("structs", [])} + if truth_struct_names: + matched = gen_struct_names & truth_struct_names + metrics["struct_match_rate"] = len(matched) / len(truth_struct_names) + metrics["struct_matched"] = sorted(matched) + metrics["struct_missed"] = sorted(truth_struct_names - gen_struct_names) + metrics["struct_extra"] = sorted(gen_struct_names - truth_struct_names) + else: + metrics["struct_match_rate"] = 1.0 + + gen_fields = set() + for s in gen_elements.get("structs", []): + for f in s.get("fields", []): + gen_fields.add(f"{s['name']}.{f['name']}") + truth_fields = set() + for s in truth_elements.get("structs", []): + for f in s.get("fields", []): + truth_fields.add(f"{s['name']}.{f['name']}") + if truth_fields: + matched_fields = gen_fields & truth_fields + metrics["field_match_rate"] = len(matched_fields) / len(truth_fields) + metrics["fields_missed"] = sorted(truth_fields - gen_fields) + metrics["fields_extra"] = sorted(gen_fields - truth_fields) + else: + metrics["field_match_rate"] = 1.0 + + gen_funcs = set(gen_elements.get("functions", [])) + truth_funcs = set(truth_elements.get("functions", [])) + if truth_funcs: + metrics["func_match_rate"] = len(gen_funcs & truth_funcs) / len(truth_funcs) + else: + metrics["func_match_rate"] = 1.0 + + gen_markers = set(gen_elements.get("markers", [])) + truth_markers = set(truth_elements.get("markers", [])) + if truth_markers: + metrics["marker_match_rate"] = len(gen_markers & truth_markers) / len(truth_markers) + else: + metrics["marker_match_rate"] = 1.0 + + return metrics + + +def _detect_outperformance( + gen_elements: dict, + truth_elements: dict, + filepath: str, + gen_content: str, + truth_content: str, +) -> list[OutperformanceFinding]: + """Detect cases where generated code is better than the truth.""" + findings: list[OutperformanceFinding] = [] + + gen_markers = set(gen_elements.get("markers", [])) + truth_markers = set(truth_elements.get("markers", [])) + extra_markers = gen_markers - truth_markers + + for marker in extra_markers: + for vm in VALIDATION_MARKERS: + if vm in marker: + findings.append(OutperformanceFinding( + category="markers", + severity="actionable", + file=filepath, + description=f"Tool added validation marker missing from human impl: {marker}", + generated_snippet=marker, + )) + break + + gen_fields_with_tags: dict[str, str] = {} + for s in gen_elements.get("structs", []): + for f in s.get("fields", []): + key = f"{s['name']}.{f['name']}" + gen_fields_with_tags[key] = f.get("tags", "") + + truth_fields_with_tags: dict[str, str] = {} + for s in truth_elements.get("structs", []): + for f in s.get("fields", []): + key = f"{s['name']}.{f['name']}" + truth_fields_with_tags[key] = f.get("tags", "") + + common_fields = set(gen_fields_with_tags) & set(truth_fields_with_tags) + for field_key in common_fields: + gen_tag = gen_fields_with_tags[field_key] + truth_tag = truth_fields_with_tags[field_key] + if "validate:" in gen_tag and "validate:" not in truth_tag: + findings.append(OutperformanceFinding( + category="validation", + severity="actionable", + file=filepath, + description=f"Tool added validation tags to {field_key} that human omitted", + generated_snippet=gen_tag, + truth_snippet=truth_tag, + )) + + gen_comment_lines = len([l for l in gen_content.split("\n") if l.strip().startswith("//")]) + truth_comment_lines = len([l for l in truth_content.split("\n") if l.strip().startswith("//")]) + if gen_comment_lines > truth_comment_lines * 1.5 and gen_comment_lines - truth_comment_lines > 5: + findings.append(OutperformanceFinding( + category="docs", + severity="informational", + file=filepath, + description=( + f"Tool generated significantly more documentation " + f"({gen_comment_lines} vs {truth_comment_lines} comment lines)" + ), + )) + + return findings + + +def compare_iteration( + gen_result: GenerationResult, + truth: GroundTruth, +) -> tuple[IterationScore, list[OutperformanceFinding]]: + """Compare a single iteration's output against ground truth.""" + gen_all_files = set(gen_result.files_created + gen_result.files_modified) + truth_all_files = set(truth.files_added + truth.files_modified) + + go_truth_files = {f for f in truth_all_files if f.endswith(".go")} + go_gen_files = {f for f in gen_all_files if f.endswith(".go")} + + tp, fn, fp = _compute_file_metrics(go_gen_files, go_truth_files) + + completeness_scores: list[float] = [] + precision_scores: list[float] = [] + convention_scores: list[float] = [] + all_outperformance: list[OutperformanceFinding] = [] + + matched_files = go_gen_files & go_truth_files + for fpath in matched_files: + gen_path = gen_result.gen_dir / fpath + truth_path = truth.path / fpath + + if not gen_path.exists() or not truth_path.exists(): + continue + + gen_content = gen_path.read_text(errors="replace") + truth_content = truth_path.read_text(errors="replace") + + gen_elements = _extract_go_elements(gen_content) + truth_elements = _extract_go_elements(truth_content) + + overlap = _compute_element_overlap(gen_elements, truth_elements) + + completeness_scores.append( + (overlap.get("struct_match_rate", 0) * 0.4) + + (overlap.get("field_match_rate", 0) * 0.4) + + (overlap.get("func_match_rate", 0) * 0.2) + ) + + truth_struct_names = {s["name"] for s in truth_elements.get("structs", [])} + gen_struct_names = {s["name"] for s in gen_elements.get("structs", [])} + if gen_struct_names: + precision_scores.append( + len(gen_struct_names & truth_struct_names) / len(gen_struct_names) + ) + else: + precision_scores.append(0.0) + + convention_scores.append(overlap.get("marker_match_rate", 0)) + + outperf = _detect_outperformance( + gen_elements, truth_elements, fpath, gen_content, truth_content + ) + all_outperformance.extend(outperf) + + extra_files = gen_all_files - truth_all_files + classification = _classify_extra_files(extra_files, gen_result, truth) + + for fpath in classification.valuable_extra: + if fpath.endswith(".go"): + gen_path = gen_result.gen_dir / fpath + if gen_path.exists(): + gen_content = gen_path.read_text(errors="replace") + gen_elements = _extract_go_elements(gen_content) + if gen_elements.get("structs") or gen_elements.get("functions"): + all_outperformance.append(OutperformanceFinding( + category="extra_coverage", + severity="informational", + file=fpath, + description=( + f"Tool generated additional file with " + f"{len(gen_elements.get('structs', []))} structs, " + f"{len(gen_elements.get('functions', []))} functions " + f"that human did not create" + ), + )) + + for fpath in go_gen_files - go_truth_files: + gen_path = gen_result.gen_dir / fpath + if gen_path.exists(): + gen_content = gen_path.read_text(errors="replace") + gen_elements = _extract_go_elements(gen_content) + if gen_elements.get("structs") or gen_elements.get("functions"): + precision_scores.append(0.3) + + genuinely_wrong_go = {f for f in classification.genuinely_wrong if f.endswith(".go")} + adjusted_precision_scores = list(precision_scores) + for fpath in go_gen_files - go_truth_files: + if fpath not in genuinely_wrong_go: + continue + gen_path = gen_result.gen_dir / fpath + if gen_path.exists(): + gen_content = gen_path.read_text(errors="replace") + gen_elements = _extract_go_elements(gen_content) + if gen_elements.get("structs") or gen_elements.get("functions"): + adjusted_precision_scores.append(0.3) + + completeness = statistics.mean(completeness_scores) * 100 if completeness_scores else 0.0 + precision = statistics.mean(precision_scores) * 100 if precision_scores else 0.0 + convention = statistics.mean(convention_scores) * 100 if convention_scores else 0.0 + + num_penalizable = len(genuinely_wrong_go) + total_gen_go = len(go_gen_files) + if total_gen_go > 0: + non_penalized = total_gen_go - num_penalizable + if matched_files: + raw_match_rate = statistics.mean(precision_scores) if precision_scores else 0.0 + if non_penalized > 0: + adjusted_precision = ( + (raw_match_rate * len(matched_files) + 1.0 * len(go_gen_files - go_truth_files - genuinely_wrong_go)) + / (len(matched_files) + len(go_gen_files - go_truth_files - genuinely_wrong_go) + num_penalizable * 0.3) + ) if (len(matched_files) + len(go_gen_files - go_truth_files)) > 0 else raw_match_rate + else: + adjusted_precision = raw_match_rate + else: + adjusted_precision = 0.0 + adjusted_precision = adjusted_precision * 100 + else: + adjusted_precision = precision + + if not matched_files and go_truth_files: + completeness = 0.0 + precision = 0.0 + adjusted_precision = 0.0 + + score = IterationScore( + iteration=gen_result.iteration, + completeness=round(completeness, 1), + precision=round(precision, 1), + adjusted_precision=round(adjusted_precision, 1), + convention_compliance=round(convention, 1), + build_success=gen_result.build_success, + file_true_positives=tp, + file_false_negatives=fn, + file_false_positives=fp, + file_classification=classification, + ) + + return score, all_outperformance + + +def compare_all_iterations( + gen_results: list[GenerationResult], + truth: GroundTruth, + ep_url: str, + repo_url: str, + description: str, + implementation_prs: list[int], +) -> BenchmarkResult: + """Compare all iterations and compute aggregate + consistency metrics.""" + all_scores: list[IterationScore] = [] + all_outperformance: list[OutperformanceFinding] = [] + + for gen in gen_results: + score, outperf = compare_iteration(gen, truth) + all_scores.append(score) + all_outperformance.extend(outperf) + + dedup_findings: list[OutperformanceFinding] = [] + seen_descriptions: set[str] = set() + for f in all_outperformance: + key = f"{f.file}:{f.description}" + if key not in seen_descriptions: + seen_descriptions.add(key) + dedup_findings.append(f) + + completeness_values = [s.completeness for s in all_scores] + precision_values = [s.precision for s in all_scores] + + score_variance = {} + if len(completeness_values) > 1: + score_variance["completeness"] = round(statistics.stdev(completeness_values), 2) + score_variance["precision"] = round(statistics.stdev(precision_values), 2) + + best_iter = max(all_scores, key=lambda s: s.completeness + s.precision).iteration + + stable: list[str] = [] + unstable: list[str] = [] + + if len(gen_results) > 1: + per_iter_files = [ + set(g.files_created + g.files_modified) + for g in gen_results + ] + all_files_union = set().union(*per_iter_files) if per_iter_files else set() + for f in all_files_union: + present_count = sum(1 for pf in per_iter_files if f in pf) + if present_count == len(gen_results): + stable.append(f) + else: + unstable.append(f) + + return BenchmarkResult( + ep_url=ep_url, + repo_url=repo_url, + description=description, + implementation_prs=implementation_prs, + iteration_scores=all_scores, + outperformance_findings=dedup_findings, + stable_elements=sorted(stable), + unstable_elements=sorted(unstable), + score_variance=score_variance, + best_iteration=best_iter, + median_completeness=round(statistics.median(completeness_values), 1), + median_precision=round(statistics.median(precision_values), 1), + ) diff --git a/benchmark/config.yaml b/benchmark/config.yaml new file mode 100644 index 0000000..d7e1917 --- /dev/null +++ b/benchmark/config.yaml @@ -0,0 +1,15 @@ +benchmark_cases: + - ep_url: "https://github.com/openshift/enhancements/pull/1863" + repo_url: "https://github.com/openshift/zero-trust-workload-identity-manager" + description: "SPIRE federation support" + implementation_prs: [68, 82] + +settings: + tools_to_benchmark: + - api-generate + - api-implement + iterations: 3 + output_dir: "benchmark/results" + parallel: false + model: "claude-opus-4-6" + effort: "max" diff --git a/benchmark/go_ast_helper/go.mod b/benchmark/go_ast_helper/go.mod new file mode 100644 index 0000000..94605d3 --- /dev/null +++ b/benchmark/go_ast_helper/go.mod @@ -0,0 +1,3 @@ +module github.com/openshift-eng/oape-ai-e2e/benchmark/go_ast_helper + +go 1.24.6 diff --git a/benchmark/go_ast_helper/main.go b/benchmark/go_ast_helper/main.go new file mode 100644 index 0000000..7ed6610 --- /dev/null +++ b/benchmark/go_ast_helper/main.go @@ -0,0 +1,219 @@ +package main + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "strings" +) + +type FieldInfo struct { + Name string `json:"name"` + Type string `json:"type"` + Tags string `json:"tags,omitempty"` + Markers []string `json:"markers,omitempty"` +} + +type StructInfo struct { + Name string `json:"name"` + Fields []FieldInfo `json:"fields"` + Markers []string `json:"markers,omitempty"` +} + +type FuncInfo struct { + Name string `json:"name"` + Params []string `json:"params,omitempty"` + Returns []string `json:"returns,omitempty"` +} + +type ConstInfo struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Value string `json:"value,omitempty"` +} + +type ExtractResult struct { + Structs []StructInfo `json:"structs"` + Functions []FuncInfo `json:"functions"` + Markers []string `json:"markers"` + Consts []ConstInfo `json:"consts"` +} + +func extractFile(filename string) (*ExtractResult, error) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parsing %s: %w", filename, err) + } + + result := &ExtractResult{} + + src, err := os.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", filename, err) + } + lines := strings.Split(string(src), "\n") + + getMarkersBefore := func(pos token.Pos) []string { + lineNum := fset.Position(pos).Line - 1 + var markers []string + for i := lineNum - 1; i >= 0; i-- { + trimmed := strings.TrimSpace(lines[i]) + if strings.HasPrefix(trimmed, "// +") { + markers = append(markers, strings.TrimPrefix(trimmed, "// ")) + } else if trimmed == "" || strings.HasPrefix(trimmed, "//") { + continue + } else { + break + } + } + return markers + } + + ast.Inspect(file, func(n ast.Node) bool { + switch node := n.(type) { + case *ast.GenDecl: + if node.Tok == token.TYPE { + for _, spec := range node.Specs { + ts, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + st, ok := ts.Type.(*ast.StructType) + if !ok { + continue + } + + info := StructInfo{ + Name: ts.Name.Name, + Markers: getMarkersBefore(node.Pos()), + } + + if st.Fields != nil { + for _, field := range st.Fields.List { + if len(field.Names) == 0 { + continue + } + fi := FieldInfo{ + Name: field.Names[0].Name, + Type: exprToString(field.Type), + Markers: getMarkersBefore(field.Pos()), + } + if field.Tag != nil { + fi.Tags = field.Tag.Value + } + info.Fields = append(info.Fields, fi) + } + } + + result.Structs = append(result.Structs, info) + } + } + + if node.Tok == token.CONST { + for _, spec := range node.Specs { + vs, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for i, name := range vs.Names { + ci := ConstInfo{Name: name.Name} + if vs.Type != nil { + ci.Type = exprToString(vs.Type) + } + if i < len(vs.Values) { + ci.Value = exprToString(vs.Values[i]) + } + result.Consts = append(result.Consts, ci) + } + } + } + + case *ast.FuncDecl: + fi := FuncInfo{Name: node.Name.Name} + if node.Type.Params != nil { + for _, p := range node.Type.Params.List { + ptype := exprToString(p.Type) + for _, name := range p.Names { + fi.Params = append(fi.Params, name.Name+" "+ptype) + } + if len(p.Names) == 0 { + fi.Params = append(fi.Params, ptype) + } + } + } + if node.Type.Results != nil { + for _, r := range node.Type.Results.List { + fi.Returns = append(fi.Returns, exprToString(r.Type)) + } + } + result.Functions = append(result.Functions, fi) + } + return true + }) + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "// +kubebuilder:") || strings.HasPrefix(trimmed, "// +optional") { + result.Markers = append(result.Markers, strings.TrimPrefix(trimmed, "// ")) + } + } + + return result, nil +} + +func exprToString(expr ast.Expr) string { + switch e := expr.(type) { + case *ast.Ident: + return e.Name + case *ast.StarExpr: + return "*" + exprToString(e.X) + case *ast.SelectorExpr: + return exprToString(e.X) + "." + e.Sel.Name + case *ast.ArrayType: + return "[]" + exprToString(e.Elt) + case *ast.MapType: + return "map[" + exprToString(e.Key) + "]" + exprToString(e.Value) + case *ast.BasicLit: + return e.Value + default: + return fmt.Sprintf("%T", expr) + } +} + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, "Usage: %s extract --file \n", os.Args[0]) + os.Exit(1) + } + + cmd := os.Args[1] + if cmd != "extract" { + fmt.Fprintf(os.Stderr, "Unknown command: %s\n", cmd) + os.Exit(1) + } + + var filename string + for i, arg := range os.Args[2:] { + if arg == "--file" && i+3 < len(os.Args) { + filename = os.Args[i+3] + } + } + if filename == "" { + fmt.Fprintln(os.Stderr, "Missing --file argument") + os.Exit(1) + } + + result, err := extractFile(filename) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + enc.Encode(result) +} diff --git a/benchmark/ground_truth.py b/benchmark/ground_truth.py new file mode 100644 index 0000000..c7c18d0 --- /dev/null +++ b/benchmark/ground_truth.py @@ -0,0 +1,96 @@ +"""Ground truth extraction from merged implementation PRs. + +Clones the repo at the final merge state and extracts the combined diff +across all implementation PRs relative to the pre-EP baseline. +""" + +import logging +import shutil +import subprocess +import tempfile +from pathlib import Path + +from models import GroundTruth, PRTimeline + +logger = logging.getLogger(__name__) + + +def _run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + logger.debug("Running: %s", " ".join(cmd)) + return subprocess.run(cmd, capture_output=True, text=True, check=True, **kwargs) + + +def extract_combined_truth(repo_url: str, timeline: PRTimeline) -> GroundTruth: + """Extract the combined ground truth from all implementation PRs. + + Clones the repo into a separate temp directory, computes + `git diff earliest_parent..latest_merge`, and categorises files. + """ + truth_dir = Path(tempfile.mkdtemp(prefix="oape-bench-truth-")) + output_dir = Path(tempfile.mkdtemp(prefix="oape-bench-truth-files-")) + + baseline = timeline.earliest_parent_sha + final = timeline.latest_merge_sha + + logger.info("Cloning %s for ground truth extraction...", repo_url) + _run(["git", "clone", "--quiet", repo_url, str(truth_dir)]) + + logger.info("Computing combined diff %s..%s", baseline[:12], final[:12]) + diff_result = _run( + ["git", "diff", f"{baseline}..{final}"], + cwd=truth_dir, + ) + combined_diff = diff_result.stdout + + stat_result = _run( + ["git", "diff", "--name-status", f"{baseline}..{final}"], + cwd=truth_dir, + ) + + files_added: list[str] = [] + files_modified: list[str] = [] + + for line in stat_result.stdout.strip().split("\n"): + if not line.strip(): + continue + parts = line.split("\t", 1) + if len(parts) < 2: + continue + status, filepath = parts[0].strip(), parts[1].strip() + if status == "A": + files_added.append(filepath) + elif status in ("M", "R", "C"): + files_modified.append(filepath) + + _run(["git", "checkout", "--quiet", final], cwd=truth_dir) + diff_hunks: dict[str, str] = {} + + for fpath in files_added + files_modified: + src = truth_dir / fpath + if src.exists(): + dst = output_dir / fpath + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + try: + hunk_result = _run( + ["git", "diff", f"{baseline}..{final}", "--", fpath], + cwd=truth_dir, + ) + diff_hunks[fpath] = hunk_result.stdout + except subprocess.CalledProcessError: + logger.warning("Could not extract diff hunks for %s", fpath) + + shutil.rmtree(truth_dir, ignore_errors=True) + + logger.info( + "Ground truth extracted: %d added, %d modified files", + len(files_added), len(files_modified), + ) + return GroundTruth( + path=output_dir, + combined_diff=combined_diff, + files_added=files_added, + files_modified=files_modified, + diff_hunks=diff_hunks, + ) diff --git a/benchmark/isolate.py b/benchmark/isolate.py new file mode 100644 index 0000000..32df3bd --- /dev/null +++ b/benchmark/isolate.py @@ -0,0 +1,126 @@ +"""Bias-free isolation engine for benchmark runs. + +Resolves multi-PR timelines and creates sealed generation environments +where the EP implementation code does not exist. +""" + +import json +import logging +import subprocess +import tempfile +from pathlib import Path + +from models import IsolatedEnv, PRDetail, PRTimeline + +logger = logging.getLogger(__name__) + + +def _run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + logger.debug("Running: %s", " ".join(cmd)) + return subprocess.run(cmd, capture_output=True, text=True, check=True, **kwargs) + + +def _gh_json(args: list[str]) -> dict | list: + result = _run(["gh"] + args) + return json.loads(result.stdout) + + +def _extract_repo_nwo(repo_url: str) -> str: + """Extract owner/repo from a GitHub URL.""" + url = repo_url.rstrip("/").removesuffix(".git") + parts = url.split("/") + return f"{parts[-2]}/{parts[-1]}" + + +def resolve_pr_timeline(repo_url: str, pr_numbers: list[int]) -> PRTimeline: + """Fetch merge info for each PR and build a sorted timeline. + + Returns the earliest parent SHA (pre-EP baseline) and latest merge SHA + (complete implementation), plus all changed files. + """ + nwo = _extract_repo_nwo(repo_url) + details: list[PRDetail] = [] + + for pr_num in pr_numbers: + logger.info("Fetching PR #%d from %s...", pr_num, nwo) + + pr_data = _gh_json([ + "pr", "view", "--repo", nwo, str(pr_num), + "--json", "mergeCommit,files,mergedAt", + ]) + + merge_sha = pr_data["mergeCommit"]["oid"] + merged_at = pr_data["mergedAt"] + files = [f["path"] for f in pr_data.get("files", [])] + + commit_data = _gh_json([ + "api", f"repos/{nwo}/commits/{merge_sha}", + ]) + parent_sha = commit_data["parents"][0]["sha"] + + details.append(PRDetail( + number=pr_num, + merge_sha=merge_sha, + parent_sha=parent_sha, + merged_at=merged_at, + files=files, + )) + + details.sort(key=lambda d: d.merged_at) + + all_files = [] + seen = set() + for d in details: + for f in d.files: + if f not in seen: + all_files.append(f) + seen.add(f) + + return PRTimeline( + earliest_parent_sha=details[0].parent_sha, + latest_merge_sha=details[-1].merge_sha, + all_changed_files=all_files, + pr_details=details, + ) + + +def prepare_generation_env(repo_url: str, timeline: PRTimeline) -> IsolatedEnv: + """Clone the repo at the pre-EP baseline commit in an isolated temp dir. + + Runs verification checks to ensure no EP code is present. + """ + gen_dir = Path(tempfile.mkdtemp(prefix="oape-bench-gen-")) + baseline = timeline.earliest_parent_sha + + logger.info("Cloning %s at %s into %s", repo_url, baseline[:12], gen_dir) + _run(["git", "clone", "--quiet", repo_url, str(gen_dir)]) + _run(["git", "checkout", "--quiet", baseline], cwd=gen_dir) + + warnings: list[str] = [] + + merge_shas = {d.merge_sha for d in timeline.pr_details} + try: + log_result = _run(["git", "log", "--oneline", "--format=%H"], cwd=gen_dir) + commit_history = set(log_result.stdout.strip().split("\n")) + leaked = merge_shas & commit_history + if leaked: + msg = f"BIAS WARNING: merge commits found in history: {leaked}" + logger.warning(msg) + warnings.append(msg) + except subprocess.CalledProcessError: + warnings.append("Could not verify git log for merge commit absence") + + for fpath in timeline.all_changed_files: + full = gen_dir / fpath + if not full.exists(): + continue + if fpath.endswith("_types.go") or "types_" in fpath: + logger.info("Baseline file exists (expected for modifications): %s", fpath) + + logger.info("Isolated environment ready at %s (baseline %s)", gen_dir, baseline[:12]) + return IsolatedEnv( + path=gen_dir, + baseline_sha=baseline, + all_changed_files=timeline.all_changed_files, + warnings=warnings, + ) diff --git a/benchmark/models.py b/benchmark/models.py new file mode 100644 index 0000000..3f52857 --- /dev/null +++ b/benchmark/models.py @@ -0,0 +1,136 @@ +"""Shared data models for the benchmark pipeline.""" + +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class PRDetail: + number: int + merge_sha: str + parent_sha: str + merged_at: str + files: list[str] + + +@dataclass +class PRTimeline: + earliest_parent_sha: str + latest_merge_sha: str + all_changed_files: list[str] + pr_details: list[PRDetail] + + +@dataclass +class IsolatedEnv: + path: Path + baseline_sha: str + all_changed_files: list[str] + warnings: list[str] = field(default_factory=list) + + +@dataclass +class GroundTruth: + path: Path + combined_diff: str + files_added: list[str] + files_modified: list[str] + diff_hunks: dict[str, str] = field(default_factory=dict) + + +@dataclass +class GenerationResult: + iteration: int + diff: str + files_created: list[str] + files_modified: list[str] + build_success: bool + gen_dir: Path + agent_log: str = "" + cost_usd: float = 0.0 + tool_version: str = "original" # "original", "improved-v1", "improved-v2", ... + + +@dataclass +class ToolImprovement: + iteration: int + files_changed: dict[str, str] # filename -> unified diff of changes + analysis_summary: str = "" + improvement_cost_usd: float = 0.0 + + +@dataclass +class ASTElement: + name: str + kind: str # "struct", "field", "function", "const", "marker" + details: dict = field(default_factory=dict) + + +@dataclass +class OutperformanceFinding: + category: str # "markers", "validation", "naming", "docs", "tests" + severity: str # "informational", "actionable" + file: str + description: str + generated_snippet: str = "" + truth_snippet: str = "" + + +@dataclass +class FileClassification: + """Classification of extra files (generated but not in ground truth).""" + auto_generated: list[str] = field(default_factory=list) + formatting_only: list[str] = field(default_factory=list) + valuable_extra: list[str] = field(default_factory=list) + genuinely_wrong: list[str] = field(default_factory=list) + + +@dataclass +class IterationScore: + iteration: int + completeness: float + precision: float # raw precision (all extras penalized) + adjusted_precision: float # excludes auto-generated and formatting-only extras + convention_compliance: float + build_success: bool + file_true_positives: int = 0 + file_false_negatives: int = 0 + file_false_positives: int = 0 + file_classification: FileClassification = field(default_factory=FileClassification) + + +@dataclass +class BenchmarkResult: + ep_url: str + repo_url: str + description: str + implementation_prs: list[int] + iteration_scores: list[IterationScore] = field(default_factory=list) + outperformance_findings: list[OutperformanceFinding] = field(default_factory=list) + stable_elements: list[str] = field(default_factory=list) + unstable_elements: list[str] = field(default_factory=list) + score_variance: dict[str, float] = field(default_factory=dict) + best_iteration: int = 0 + median_completeness: float = 0.0 + median_precision: float = 0.0 + tool_improvements: list[ToolImprovement] = field(default_factory=list) + mode: str = "feedback_loop" # "feedback_loop" or "measurement" + + +@dataclass +class BenchmarkCase: + ep_url: str + repo_url: str + description: str + implementation_prs: list[int] + + +@dataclass +class BenchmarkConfig: + cases: list[BenchmarkCase] + tools_to_benchmark: list[str] + iterations: int + output_dir: str + parallel: bool + model: str = "claude-opus-4-6" + effort: str = "max" diff --git a/benchmark/report.py b/benchmark/report.py new file mode 100644 index 0000000..7648d4d --- /dev/null +++ b/benchmark/report.py @@ -0,0 +1,353 @@ +"""Report generation for benchmark results. + +Produces per-EP markdown reports and aggregate summaries. +""" + +import json +import logging +from datetime import datetime, timezone +from pathlib import Path + +from models import BenchmarkResult, GenerationResult, GroundTruth + +logger = logging.getLogger(__name__) + + +def _repo_short_name(repo_url: str) -> str: + return repo_url.rstrip("/").split("/")[-1].removesuffix(".git") + + +def _ep_number(ep_url: str) -> str: + return ep_url.rstrip("/").split("/")[-1] + + +def generate_ep_report( + result: BenchmarkResult, + gen_results: list[GenerationResult], + truth: GroundTruth, + output_dir: Path, +) -> Path: + """Generate a per-EP benchmark report.""" + repo_name = _repo_short_name(result.repo_url) + ep_num = _ep_number(result.ep_url) + report_dir = output_dir / repo_name / f"ep-{ep_num}" + report_dir.mkdir(parents=True, exist_ok=True) + + lines: list[str] = [] + lines.append(f"# Benchmark Report: EP #{ep_num}") + lines.append("") + lines.append(f"**EP**: [{result.ep_url}]({result.ep_url})") + lines.append(f"**Repository**: `{result.repo_url}`") + lines.append(f"**Description**: {result.description}") + lines.append(f"**Implementation PRs**: {', '.join(f'#{pr}' for pr in result.implementation_prs)}") + lines.append(f"**Iterations**: {len(result.iteration_scores)}") + lines.append(f"**Generated**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") + lines.append("") + + is_feedback = result.mode == "feedback_loop" + lines.append(f"**Mode**: {'Feedback Loop (tool improved between iterations)' if is_feedback else 'Measurement (same tool each iteration)'}") + lines.append("") + + lines.append("## Score Progression") + lines.append("") + lines.append("| Iteration | Tool Version | Completeness | Raw Precision | Adjusted Precision | Convention | Build |") + lines.append("|-----------|-------------|-------------|--------------|-------------------|------------|-------|") + for idx, s in enumerate(result.iteration_scores): + build = "PASS" if s.build_success else "FAIL" + tool_ver = gen_results[idx].tool_version if idx < len(gen_results) else "?" + lines.append( + f"| {s.iteration} | {tool_ver} | {s.completeness:.1f}% | {s.precision:.1f}% | " + f"{s.adjusted_precision:.1f}% | {s.convention_compliance:.1f}% | {build} |" + ) + lines.append("") + lines.append("> **Raw Precision**: penalizes ALL extra files equally. " + "**Adjusted Precision**: excludes auto-generated artifacts and formatting-only changes, " + "and does not penalize valuable extra files (tests, validation, new types).") + lines.append("") + + lines.append("### Summary") + lines.append("") + lines.append(f"- **Best Iteration**: {result.best_iteration}") + if len(result.iteration_scores) >= 2: + first = result.iteration_scores[0] + last = result.iteration_scores[-1] + lines.append(f"- **Completeness**: {first.completeness:.1f}% -> {last.completeness:.1f}% ({last.completeness - first.completeness:+.1f}%)") + lines.append(f"- **Adjusted Precision**: {first.adjusted_precision:.1f}% -> {last.adjusted_precision:.1f}% ({last.adjusted_precision - first.adjusted_precision:+.1f}%)") + total_gen_cost = sum(g.cost_usd for g in gen_results) + total_imp_cost = sum(imp.improvement_cost_usd for imp in result.tool_improvements) + lines.append(f"- **Total Cost**: ${total_gen_cost + total_imp_cost:.2f} (generation: ${total_gen_cost:.2f}, improvement: ${total_imp_cost:.2f})") + lines.append("") + + for idx, s in enumerate(result.iteration_scores): + fc = s.file_classification + if fc.auto_generated or fc.formatting_only or fc.valuable_extra or fc.genuinely_wrong: + tool_ver = gen_results[idx].tool_version if idx < len(gen_results) else "?" + lines.append(f"### Iteration {s.iteration} ({tool_ver}) - Extra File Breakdown") + lines.append("") + if fc.auto_generated: + lines.append(f"**Auto-generated artifacts** ({len(fc.auto_generated)} files) -- from `make generate`/`make manifests`, not tool errors:") + for f in fc.auto_generated: + lines.append(f"- `{f}`") + lines.append("") + if fc.formatting_only: + lines.append(f"**Formatting-only changes** ({len(fc.formatting_only)} files) -- whitespace/import reordering, no behavior change:") + for f in fc.formatting_only: + lines.append(f"- `{f}`") + lines.append("") + if fc.valuable_extra: + lines.append(f"**Valuable extras** ({len(fc.valuable_extra)} files) -- tool generated useful code the human didn't:") + for f in fc.valuable_extra: + lines.append(f"- `{f}`") + lines.append("") + if fc.genuinely_wrong: + lines.append(f"**Genuinely wrong** ({len(fc.genuinely_wrong)} files) -- files that should not have been touched:") + for f in fc.genuinely_wrong: + lines.append(f"- `{f}`") + lines.append("") + + if result.stable_elements or result.unstable_elements: + lines.append("## Consistency Analysis") + lines.append("") + if result.stable_elements: + lines.append(f"### Stable Elements ({len(result.stable_elements)} files produced in ALL iterations)") + lines.append("") + for el in result.stable_elements: + lines.append(f"- `{el}`") + lines.append("") + if result.unstable_elements: + lines.append(f"### Unstable Elements ({len(result.unstable_elements)} files produced in SOME iterations)") + lines.append("") + for el in result.unstable_elements: + lines.append(f"- `{el}`") + lines.append("") + + if result.outperformance_findings: + lines.append("## Tool Outperformed Human") + lines.append("") + lines.append("Cases where the generated code was arguably better than the shipped implementation:") + lines.append("") + + actionable = [f for f in result.outperformance_findings if f.severity == "actionable"] + informational = [f for f in result.outperformance_findings if f.severity == "informational"] + + if actionable: + lines.append("### Actionable Improvements") + lines.append("") + for f in actionable: + lines.append(f"- **[{f.category}]** `{f.file}`: {f.description}") + if f.generated_snippet: + lines.append(f" - Generated: `{f.generated_snippet}`") + if f.truth_snippet: + lines.append(f" - Actual: `{f.truth_snippet}`") + lines.append("") + + if informational: + lines.append("### Informational") + lines.append("") + for f in informational: + lines.append(f"- **[{f.category}]** `{f.file}`: {f.description}") + lines.append("") + + if result.tool_improvements: + lines.append("## Tool Improvements Between Iterations") + lines.append("") + for imp in result.tool_improvements: + lines.append(f"### After Iteration {imp.iteration}") + lines.append("") + if imp.analysis_summary: + lines.append(f"**Analysis**: {imp.analysis_summary[:500]}") + lines.append("") + if imp.files_changed: + for fname, diff_text in imp.files_changed.items(): + added = sum(1 for l in diff_text.split("\n") if l.startswith("+") and not l.startswith("+++")) + removed = sum(1 for l in diff_text.split("\n") if l.startswith("-") and not l.startswith("---")) + lines.append(f"**`{fname}`** (+{added}/-{removed} lines)") + lines.append("") + diff_preview = diff_text[:3000] + lines.append("```diff") + lines.append(diff_preview) + if len(diff_text) > 3000: + lines.append(f"... ({len(diff_text) - 3000} more chars truncated)") + lines.append("```") + lines.append("") + else: + lines.append("No tool file changes in this iteration.") + lines.append("") + if imp.improvement_cost_usd > 0: + lines.append(f"*Improvement cost: ${imp.improvement_cost_usd:.2f}*") + lines.append("") + + lines.append("## Ground Truth Files") + lines.append("") + if truth.files_added: + lines.append(f"### Added ({len(truth.files_added)})") + for f in truth.files_added: + lines.append(f"- `{f}`") + lines.append("") + if truth.files_modified: + lines.append(f"### Modified ({len(truth.files_modified)})") + for f in truth.files_modified: + lines.append(f"- `{f}`") + lines.append("") + + best = next((g for g in gen_results if g.iteration == result.best_iteration), gen_results[0]) + lines.append(f"## Best Iteration (#{best.iteration}) Files") + lines.append("") + if best.files_created: + lines.append(f"### Created ({len(best.files_created)})") + for f in best.files_created: + lines.append(f"- `{f}`") + lines.append("") + if best.files_modified: + lines.append(f"### Modified ({len(best.files_modified)})") + for f in best.files_modified: + lines.append(f"- `{f}`") + lines.append("") + + report_path = report_dir / "report.md" + report_path.write_text("\n".join(lines)) + + json_path = report_dir / "report.json" + json_data = { + "ep_url": result.ep_url, + "repo_url": result.repo_url, + "description": result.description, + "implementation_prs": result.implementation_prs, + "median_completeness": result.median_completeness, + "median_precision": result.median_precision, + "best_iteration": result.best_iteration, + "score_variance": result.score_variance, + "iteration_scores": [ + { + "iteration": s.iteration, + "completeness": s.completeness, + "precision": s.precision, + "adjusted_precision": s.adjusted_precision, + "convention_compliance": s.convention_compliance, + "build_success": s.build_success, + "file_tp": s.file_true_positives, + "file_fn": s.file_false_negatives, + "file_fp": s.file_false_positives, + "file_classification": { + "auto_generated": s.file_classification.auto_generated, + "formatting_only": s.file_classification.formatting_only, + "valuable_extra": s.file_classification.valuable_extra, + "genuinely_wrong": s.file_classification.genuinely_wrong, + }, + } + for s in result.iteration_scores + ], + "outperformance_count": len(result.outperformance_findings), + "stable_files": len(result.stable_elements), + "unstable_files": len(result.unstable_elements), + "mode": result.mode, + "tool_improvements": [ + { + "after_iteration": imp.iteration, + "files_changed": list(imp.files_changed.keys()), + "improvement_cost_usd": imp.improvement_cost_usd, + } + for imp in result.tool_improvements + ], + } + json_path.write_text(json.dumps(json_data, indent=2)) + + for gen in gen_results: + iter_dir = report_dir / f"iter-{gen.iteration}" + iter_dir.mkdir(parents=True, exist_ok=True) + (iter_dir / "diff.patch").write_text(gen.diff) + (iter_dir / "scores.json").write_text(json.dumps({ + "iteration": gen.iteration, + "files_created": gen.files_created, + "files_modified": gen.files_modified, + "build_success": gen.build_success, + "cost_usd": gen.cost_usd, + }, indent=2)) + + truth_out = report_dir / "truth" + truth_out.mkdir(parents=True, exist_ok=True) + (truth_out / "combined.diff").write_text(truth.combined_diff) + (truth_out / "files_added.txt").write_text("\n".join(truth.files_added)) + (truth_out / "files_modified.txt").write_text("\n".join(truth.files_modified)) + + for imp in result.tool_improvements: + imp_dir = report_dir / f"tool-improvement-after-iter-{imp.iteration}" + imp_dir.mkdir(parents=True, exist_ok=True) + for fname, diff_text in imp.files_changed.items(): + safe_name = fname.replace("/", "__") + (imp_dir / f"{safe_name}.diff").write_text(diff_text) + + logger.info("Report written to %s", report_path) + return report_path + + +def generate_aggregate_report( + results: list[BenchmarkResult], + output_dir: Path, +) -> Path: + """Generate an aggregate report across all benchmark cases.""" + lines: list[str] = [] + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + + lines.append("# OAPE Benchmark Aggregate Report") + lines.append("") + lines.append(f"**Generated**: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") + lines.append(f"**Total EPs**: {len(results)}") + lines.append("") + + lines.append("## Summary Table") + lines.append("") + lines.append("| EP | Repo | Completeness | Precision | Best Iter | Consistency |") + lines.append("|----|------|-------------|-----------|-----------|-------------|") + for r in results: + ep_num = _ep_number(r.ep_url) + repo = _repo_short_name(r.repo_url) + var = r.score_variance.get("completeness", 0) + consistency = "High" if var < 5 else ("Medium" if var < 15 else "Low") + lines.append( + f"| #{ep_num} | {repo} | {r.median_completeness:.1f}% | " + f"{r.median_precision:.1f}% | {r.best_iteration} | {consistency} |" + ) + lines.append("") + + if len(results) > 1: + all_completeness = [r.median_completeness for r in results] + all_precision = [r.median_precision for r in results] + import statistics + lines.append("### Averages Across All EPs") + lines.append("") + lines.append(f"- **Mean Completeness**: {statistics.mean(all_completeness):.1f}%") + lines.append(f"- **Mean Precision**: {statistics.mean(all_precision):.1f}%") + lines.append("") + + total_outperf = sum(len(r.outperformance_findings) for r in results) + if total_outperf > 0: + lines.append("## Tool Outperformed Human") + lines.append("") + lines.append(f"Total findings across all EPs: **{total_outperf}**") + lines.append("") + + category_counts: dict[str, int] = {} + for r in results: + for f in r.outperformance_findings: + category_counts[f.category] = category_counts.get(f.category, 0) + 1 + + lines.append("| Category | Count |") + lines.append("|----------|-------|") + for cat, count in sorted(category_counts.items(), key=lambda x: -x[1]): + lines.append(f"| {cat} | {count} |") + lines.append("") + + lines.append("## Consistency Ranking") + lines.append("") + sorted_by_consistency = sorted(results, key=lambda r: r.score_variance.get("completeness", 0)) + for r in sorted_by_consistency: + ep_num = _ep_number(r.ep_url) + var = r.score_variance.get("completeness", 0) + lines.append(f"- EP #{ep_num}: stddev={var:.2f} ({len(r.stable_elements)} stable, {len(r.unstable_elements)} unstable files)") + lines.append("") + + report_path = output_dir / f"aggregate-{ts}.md" + report_path.write_text("\n".join(lines)) + logger.info("Aggregate report written to %s", report_path) + return report_path diff --git a/benchmark/requirements.txt b/benchmark/requirements.txt new file mode 100644 index 0000000..a84fb88 --- /dev/null +++ b/benchmark/requirements.txt @@ -0,0 +1,3 @@ +pyyaml>=6.0 +claude-agent-sdk>=0.1.0 +rich>=13.0 diff --git a/benchmark/runner.py b/benchmark/runner.py new file mode 100644 index 0000000..cfbe24d --- /dev/null +++ b/benchmark/runner.py @@ -0,0 +1,545 @@ +"""Benchmark runner with iterative feedback loop. + +Each iteration: generate code -> compare with ground truth -> improve the OAPE +tool (commands/skills) -> re-generate with improved tool. The tool itself gets +better after each iteration. + +All agents use claude-opus-4-6 in max effort mode. +""" + +import json +import logging +import shutil +import subprocess +import tempfile +import traceback +from pathlib import Path + +from compare import compare_iteration +from models import GenerationResult, GroundTruth, IsolatedEnv, IterationScore, ToolImprovement + +logger = logging.getLogger(__name__) + +PLUGIN_DIR = str(Path(__file__).resolve().parent.parent / "plugins" / "oape") + +TOOL_FILES = [ + "commands/api-generate.md", + "commands/api-implement.md", +] + + +def _run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, check=True, **kwargs) + + +def _clone_iteration_env(source_env: IsolatedEnv, iteration: int) -> Path: + """Create a fresh copy of the isolated environment for one iteration.""" + iter_dir = Path(tempfile.mkdtemp(prefix=f"oape-bench-iter{iteration}-")) + logger.info("Copying baseline to %s for iteration %d", iter_dir, iteration) + shutil.copytree(source_env.path, iter_dir, dirs_exist_ok=True) + _run(["git", "checkout", "--quiet", source_env.baseline_sha], cwd=iter_dir) + _run(["git", "clean", "-fdx", "--quiet"], cwd=iter_dir) + return iter_dir + + +def _build_generation_prompt(ep_url: str, repo_dir: str) -> str: + return f"""You are running OAPE tools to implement an Enhancement Proposal. + +Repository: {repo_dir} (already cloned and ready) +EP URL: {ep_url} + +Steps: +1. Run /oape:api-generate {ep_url} +2. After api-generate completes, run /oape:api-implement {ep_url} + +CRITICAL RULES: +- Do NOT run `git push` under any circumstances +- Do NOT run `gh pr create` under any circumstances +- Do NOT create any branches for pushing +- Work ONLY in the current directory +- After running both commands, stop. Do not push or create PRs. +- Execute both commands fully and autonomously without asking for confirmation. +""" + + +def _try_build(work_dir: Path) -> bool: + """Attempt `make build` in the working directory.""" + try: + result = subprocess.run( + ["make", "build"], + cwd=work_dir, + capture_output=True, + text=True, + timeout=300, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + +def _backup_tool_files(plugin_dir: str, backup_dir: Path, label: str) -> None: + """Save a copy of all OAPE tool files to a backup directory.""" + dest = backup_dir / label + dest.mkdir(parents=True, exist_ok=True) + for rel in TOOL_FILES: + src = Path(plugin_dir) / rel + if src.exists(): + dst = dest / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + logger.info("Tool files backed up to %s (%s)", dest, label) + + +def _diff_tool_files(plugin_dir: str, backup_dir: Path, before_label: str) -> dict[str, str]: + """Compute unified diffs between backed-up version and current tool files.""" + diffs: dict[str, str] = {} + for rel in TOOL_FILES: + before = backup_dir / before_label / rel + current = Path(plugin_dir) / rel + if not before.exists() or not current.exists(): + continue + try: + result = subprocess.run( + ["diff", "-u", str(before), str(current)], + capture_output=True, text=True, + ) + if result.stdout.strip(): + diffs[rel] = result.stdout + except FileNotFoundError: + pass + return diffs + + +def _build_improvement_prompt( + iteration: int, + score: IterationScore, + gen_result: GenerationResult, + truth: GroundTruth, + plugin_dir: str, +) -> str: + """Build the prompt for the improver agent.""" + tool_contents: dict[str, str] = {} + for rel in TOOL_FILES: + path = Path(plugin_dir) / rel + if path.exists(): + tool_contents[rel] = path.read_text() + + truth_files_summary = [] + for f in truth.files_added[:20]: + truth_files_summary.append(f" ADDED: {f}") + for f in truth.files_modified[:20]: + truth_files_summary.append(f" MODIFIED: {f}") + + gen_files_summary = [] + for f in gen_result.files_created[:20]: + gen_files_summary.append(f" CREATED: {f}") + for f in gen_result.files_modified[:20]: + gen_files_summary.append(f" MODIFIED: {f}") + + truth_diff_preview = truth.combined_diff[:12000] + gen_diff_preview = gen_result.diff[:12000] + + fc = score.file_classification + classification_summary = [] + if fc.auto_generated: + classification_summary.append(f" Auto-generated artifacts (NOT errors): {fc.auto_generated}") + if fc.formatting_only: + classification_summary.append(f" Formatting-only changes (NOT errors): {fc.formatting_only}") + if fc.valuable_extra: + classification_summary.append(f" Valuable extras (tool outperformed human): {fc.valuable_extra}") + if fc.genuinely_wrong: + classification_summary.append(f" GENUINELY WRONG (these are the real errors): {fc.genuinely_wrong}") + classification_text = chr(10).join(classification_summary) if classification_summary else " No extra files generated." + + tool_sections = [] + for rel, content in tool_contents.items(): + tool_sections.append(f"### FILE: {rel}\n```markdown\n{content}\n```") + + return f"""You are an expert at improving AI code generation tools for OpenShift operators. + +You are given a comparison between what an OAPE tool generated vs what a human actually +shipped for a specific Enhancement Proposal. Your job is to improve the tool's instruction +files so it performs better on ALL future EPs -- not just this specific one. + +## CRITICAL: Make GENERIC improvements only + +- DO NOT add EP-specific guidance (e.g., "when generating federation support, do X"). +- DO NOT reference specific struct names, field names, or feature names from this EP. +- DO add GENERAL PATTERNS that would have caught the issues seen here. +- DO improve the tool's ability to discover what files to generate from ANY EP. +- Think: "What general rule or pattern was missing that caused this gap?" + +## Iteration {iteration} Results + +### Scores +- Completeness: {score.completeness:.1f}% (what % of ground truth structs/fields/functions were generated) +- Raw Precision: {score.precision:.1f}% (penalizes ALL extra files) +- Adjusted Precision: {score.adjusted_precision:.1f}% (excludes auto-generated artifacts and valuable extras) +- Convention Compliance: {score.convention_compliance:.1f}% +- Build Success: {"PASS" if score.build_success else "FAIL"} +- Files: {score.file_true_positives} matched, {score.file_false_negatives} missed, {score.file_false_positives} extra + +### Understanding Extra Files + +Not all "extra" files are errors: + +{classification_text} + +Files like `zz_generated.deepcopy.go`, CRD YAMLs, `bundle.Dockerfile` are auto-generated +by `make` commands -- generating them is CORRECT. Test files, sample configs, and dedicated +handler files may be the tool doing BETTER than the human. + +Only "GENUINELY WRONG" files are actual errors. + +### Ground Truth (what the human produced) +{chr(10).join(truth_files_summary)} + +Ground truth diff (first 12000 chars): +```diff +{truth_diff_preview} +``` + +### Generated Output (what the tool produced) +{chr(10).join(gen_files_summary)} + +Generated diff (first 12000 chars): +```diff +{gen_diff_preview} +``` + +## Current OAPE Tool Instructions + +{chr(10).join(tool_sections)} + +## Your Task + +Identify GENERAL PATTERNS the tool is missing, then make targeted edits: + +### Priority 1: General patterns for missing files +If the tool missed files (e.g., routes, validation, services, tests), add GENERIC +rules about when an EP implies these file types are needed. Example of a good generic +improvement: "When the EP describes exposing an endpoint, generate a routes.go file +following the existing naming pattern in the controller package." + +### Priority 2: General patterns for struct/field generation +If structs or fields don't match, improve the GENERAL conventions about how to derive +struct shapes from EP descriptions. + +### Priority 3: General patterns for markers and validation +If markers were missing, add GENERAL rules about which markers to always include. + +### Priority 4: General controller patterns +If controller logic was incomplete, add GENERAL reconciliation patterns. + +### What NOT to do: +- Do NOT add feature-specific rules (no "for federation..." or "for rotation...") +- Do NOT reference specific struct/field names from this EP +- Do NOT reduce file coverage -- generating extra tests/samples is GOOD +- Do NOT tell the tool to skip `make generate`/`make manifests` +- Do NOT rewrite entire files -- make surgical, targeted edits + +After editing, explain what GENERAL pattern you added and why. +""" + + +async def run_single_iteration( + ep_url: str, + source_env: IsolatedEnv, + iteration: int, + plugin_dir: str = PLUGIN_DIR, + model: str = "claude-opus-4-6", + effort: str = "max", +) -> GenerationResult: + """Run one iteration of OAPE tools in an isolated environment.""" + iter_dir = _clone_iteration_env(source_env, iteration) + + logger.info("=== Iteration %d: running OAPE tools (model=%s, effort=%s) ===", + iteration, model, effort) + + prompt = _build_generation_prompt(ep_url, str(iter_dir)) + agent_log_parts: list[str] = [] + cost_usd = 0.0 + + tool_version = "original" if iteration == 1 else f"improved-v{iteration - 1}" + + try: + from claude_agent_sdk import ( + query, + ClaudeAgentOptions, + AssistantMessage, + ResultMessage, + TextBlock, + ThinkingBlock, + ToolUseBlock, + ToolResultBlock, + ) + + options = ClaudeAgentOptions( + model=model, + effort=effort, + system_prompt=( + "You are an OpenShift operator code generation assistant. " + "Follow the workflow instructions precisely. " + "CRITICAL: Do NOT push branches. Do NOT create PRs. " + "Do NOT run git push or gh pr create. " + "Work purely locally. Execute all steps without pausing." + ), + cwd=str(iter_dir), + permission_mode="bypassPermissions", + allowed_tools=[ + "Read", "Write", "Edit", "MultiEdit", + "Bash", "Glob", "Grep", + "TodoRead", "TodoWrite", + ], + plugins=[{"type": "local", "path": plugin_dir}], + ) + + async for message in query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + agent_log_parts.append(block.text) + elif isinstance(block, ToolUseBlock): + agent_log_parts.append(f"[tool:{block.name}]") + elif isinstance(message, ResultMessage): + cost_usd = message.total_cost_usd + if message.result: + agent_log_parts.append(message.result) + + except ImportError: + logger.warning( + "claude_agent_sdk not available; running in dry-run mode. " + "Install claude-agent-sdk to run actual benchmarks." + ) + agent_log_parts.append("[DRY RUN] claude_agent_sdk not installed") + except Exception: + agent_log_parts.append(f"[ERROR] {traceback.format_exc()}") + logger.error("Iteration %d failed: %s", iteration, traceback.format_exc()) + + try: + diff_result = _run(["git", "diff", "HEAD"], cwd=iter_dir) + diff = diff_result.stdout + + status_result = _run(["git", "status", "--porcelain"], cwd=iter_dir) + untracked = [ + line[3:] for line in status_result.stdout.strip().split("\n") + if line.startswith("?? ") + ] + if untracked: + _run(["git", "add"] + untracked, cwd=iter_dir) + diff_result2 = _run(["git", "diff", "--cached"], cwd=iter_dir) + diff = diff + "\n" + diff_result2.stdout + _run(["git", "reset", "HEAD"] + untracked, cwd=iter_dir) + + diff_stat = _run( + ["git", "diff", "--name-status", "HEAD"], + cwd=iter_dir, + ) + all_status = _run(["git", "status", "--porcelain"], cwd=iter_dir) + + files_created: list[str] = [] + files_modified: list[str] = [] + + for line in diff_stat.stdout.strip().split("\n"): + if not line.strip(): + continue + parts = line.split("\t", 1) + if len(parts) < 2: + continue + status, fpath = parts[0].strip(), parts[1].strip() + if status == "A": + files_created.append(fpath) + elif status == "M": + files_modified.append(fpath) + + for line in all_status.stdout.strip().split("\n"): + if line.startswith("?? "): + fpath = line[3:] + if fpath not in files_created: + files_created.append(fpath) + + except subprocess.CalledProcessError: + diff = "" + files_created = [] + files_modified = [] + + build_ok = _try_build(iter_dir) + + logger.info( + "Iteration %d complete: %d created, %d modified, build=%s, cost=$%.4f", + iteration, len(files_created), len(files_modified), build_ok, cost_usd, + ) + + return GenerationResult( + iteration=iteration, + diff=diff, + files_created=files_created, + files_modified=files_modified, + build_success=build_ok, + gen_dir=iter_dir, + agent_log="\n".join(agent_log_parts), + cost_usd=cost_usd, + tool_version=tool_version, + ) + + +async def improve_tool( + iteration: int, + gen_result: GenerationResult, + truth: GroundTruth, + score: IterationScore, + plugin_dir: str = PLUGIN_DIR, + backup_dir: Path | None = None, + model: str = "claude-opus-4-6", + effort: str = "max", +) -> ToolImprovement: + """Analyze weaknesses and edit OAPE tool files to improve next iteration.""" + logger.info("=== Improving tool after iteration %d (model=%s, effort=%s) ===", + iteration, model, effort) + + if backup_dir: + _backup_tool_files(plugin_dir, backup_dir, f"before-improvement-{iteration}") + + prompt = _build_improvement_prompt(iteration, score, gen_result, truth, plugin_dir) + improvement_log: list[str] = [] + cost_usd = 0.0 + + try: + from claude_agent_sdk import ( + query, + ClaudeAgentOptions, + AssistantMessage, + ResultMessage, + TextBlock, + ToolUseBlock, + ) + + options = ClaudeAgentOptions( + model=model, + effort=effort, + system_prompt=( + "You are an expert at improving AI code generation tools. " + "You will analyze comparison results and make targeted edits to " + "OAPE tool instruction files (markdown) to improve code generation quality. " + "Focus on specific, actionable improvements. Do NOT rewrite entire files. " + "Make surgical edits that address observed weaknesses." + ), + cwd=plugin_dir, + permission_mode="bypassPermissions", + allowed_tools=[ + "Read", "Write", "Edit", "MultiEdit", + "Bash", "Glob", "Grep", + ], + ) + + async for message in query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + improvement_log.append(block.text) + elif isinstance(block, ToolUseBlock): + improvement_log.append(f"[tool:{block.name}]") + elif isinstance(message, ResultMessage): + cost_usd = message.total_cost_usd + if message.result: + improvement_log.append(message.result) + + except ImportError: + logger.warning("claude_agent_sdk not available for improvement step") + improvement_log.append("[DRY RUN] claude_agent_sdk not installed") + except Exception: + improvement_log.append(f"[ERROR] {traceback.format_exc()}") + logger.error("Tool improvement failed: %s", traceback.format_exc()) + + diffs = {} + if backup_dir: + diffs = _diff_tool_files(plugin_dir, backup_dir, f"before-improvement-{iteration}") + _backup_tool_files(plugin_dir, backup_dir, f"after-improvement-{iteration}") + + if diffs: + logger.info("Tool improved: %d files changed", len(diffs)) + for fname, diff in diffs.items(): + added = sum(1 for l in diff.split("\n") if l.startswith("+") and not l.startswith("+++")) + removed = sum(1 for l in diff.split("\n") if l.startswith("-") and not l.startswith("---")) + logger.info(" %s: +%d/-%d lines", fname, added, removed) + else: + logger.info("No tool file changes detected after improvement step") + + return ToolImprovement( + iteration=iteration, + files_changed=diffs, + analysis_summary="\n".join(improvement_log[-5:]) if improvement_log else "", + improvement_cost_usd=cost_usd, + ) + + +async def run_feedback_loop( + ep_url: str, + source_env: IsolatedEnv, + truth: GroundTruth, + num_iterations: int = 3, + plugin_dir: str = PLUGIN_DIR, + backup_dir: Path | None = None, + model: str = "claude-opus-4-6", + effort: str = "max", +) -> tuple[list[GenerationResult], list[ToolImprovement]]: + """Run iterative feedback loop: generate -> compare -> improve -> repeat. + + Each iteration uses the improved tool from the previous iteration. + Returns generation results and tool improvements. + """ + results: list[GenerationResult] = [] + improvements: list[ToolImprovement] = [] + + if backup_dir: + _backup_tool_files(plugin_dir, backup_dir, "original") + + for i in range(1, num_iterations + 1): + logger.info("=" * 50) + logger.info("FEEDBACK LOOP: Iteration %d of %d", i, num_iterations) + logger.info("=" * 50) + + result = await run_single_iteration( + ep_url, source_env, i, + plugin_dir=plugin_dir, model=model, effort=effort, + ) + results.append(result) + + score, outperf = compare_iteration(result, truth) + logger.info( + "Iteration %d scores: completeness=%.1f%% precision=%.1f%% build=%s", + i, score.completeness, score.precision, score.build_success, + ) + + if i < num_iterations: + logger.info("Analyzing weaknesses and improving tool...") + improvement = await improve_tool( + iteration=i, + gen_result=result, + truth=truth, + score=score, + plugin_dir=plugin_dir, + backup_dir=backup_dir, + model=model, + effort=effort, + ) + improvements.append(improvement) + logger.info( + "Tool improvement complete (cost=$%.4f). %d files changed.", + improvement.improvement_cost_usd, len(improvement.files_changed), + ) + else: + logger.info("Final iteration complete. No further improvements.") + + if backup_dir: + original_dir = backup_dir / "original" + if original_dir.exists(): + logger.info("Restoring original tool files...") + for rel in TOOL_FILES: + src = original_dir / rel + dst = Path(plugin_dir) / rel + if src.exists(): + shutil.copy2(src, dst) + logger.info("Original tool files restored.") + + return results, improvements From f4b2244fcad7ff515c4e245028dd883083c316ae Mon Sep 17 00:00:00 2001 From: Raushan Singh Date: Wed, 6 May 2026 16:16:02 +0530 Subject: [PATCH 2/6] refactor: replace precision metric with actionable file classification Remove the misleading "precision" metric that penalized the tool for generating better code (extra tests, samples, validation). Replace with clear file classification counts in the report: - Matched: files matching the human implementation - Missed: files the tool didn't generate (gaps) - Wrong: files that should not have been touched (real errors) - Extras: useful files the human didn't write (tool outperformed) - Auto: make-generated artifacts (expected behavior) This gives a much clearer picture: "1 wrong file" is more actionable than "45% precision." Co-authored-by: Cursor --- benchmark/README.md | 35 ++++++++++++++-------- benchmark/benchmark.py | 19 ++++++------ benchmark/compare.py | 67 ++++-------------------------------------- benchmark/config.yaml | 8 ++--- benchmark/models.py | 11 ++++--- benchmark/report.py | 56 +++++++++++++++++++---------------- benchmark/runner.py | 5 ++-- 7 files changed, 80 insertions(+), 121 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index e65e075..254b652 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -36,19 +36,28 @@ EP #1863 + Repo + PR numbers The generating agent never sees the existing implementation. The repo is cloned at the commit **before** the EP was merged, so the EP's code physically does not exist. The agent only receives the EP URL and generates from scratch. -### Scoring - -- **Completeness**: What % of the human's structs/fields/functions did the tool also produce? -- **Raw Precision**: What % of generated code matches the human's output? (penalizes all extras) -- **Adjusted Precision**: Same as raw but excludes auto-generated artifacts (`zz_generated`, CRDs, bundle manifests) and valuable extras (tests, sample configs) -- **Convention Compliance**: Kubebuilder marker and naming pattern match rate -- **Build Success**: Does `make build` pass? - -Extra files are classified into four categories: -- **Auto-generated**: Output of `make generate`/`make manifests` (not errors) -- **Formatting-only**: Whitespace/import reordering (not errors) -- **Valuable extras**: Tests, validation, sample configs the human didn't write (tool outperformed human) -- **Genuinely wrong**: Files that should not have been touched (real errors) +### Metrics + +| Metric | What it measures | +|--------|-----------------| +| **Completeness** | What % of the human's structs, fields, and functions did the tool also generate? Higher = tool covered more of what the human wrote. | +| **Convention** | What % of kubebuilder markers (`+kubebuilder:validation:Required`, `+optional`, etc.) match between generated and ground truth? | +| **Build** | Did `make build` pass on the generated code? | +| **Matched** | Files the tool generated that also exist in the human implementation (true positives). | +| **Missed** | Files in the human implementation that the tool did NOT generate (gaps to close). | +| **Wrong** | Files the tool touched that it should not have -- unrelated to the EP (real errors to fix). | +| **Extras** | Files the tool generated that the human didn't -- tests, samples, better code organization (tool outperformed human). | +| **Auto** | Files auto-generated by `make generate`/`make manifests` (expected build artifacts, not errors). | + +The report table looks like: + +``` +| Iteration | Tool Version | Completeness | Convention | Build | Matched | Missed | Wrong | Extras | Auto | +|-----------|-------------|-------------|------------|-------|---------|--------|-------|--------|------| +| 1 | original | 91.0% | 97.8% | PASS | 6 | 12 | 1 | 5 | 0 | +| 2 | improved-v1 | 99.5% | 97.8% | PASS | 8 | 10 | 1 | 5 | 0 | +| 3 | improved-v2 | 94.2% | 97.8% | PASS | 9 | 9 | 1 | 6 | 0 | +``` ## Prerequisites diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index dc87655..6780695 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -146,8 +146,9 @@ async def run_single_case( for s in benchmark_result.iteration_scores: tool_ver = gen_results[s.iteration - 1].tool_version if s.iteration <= len(gen_results) else "?" logger.info( - " Iter %d (%s): completeness=%.1f%% precision=%.1f%% build=%s", - s.iteration, tool_ver, s.completeness, s.precision, s.build_success, + " Iter %d (%s): completeness=%.1f%% convention=%.1f%% build=%s wrong=%d extras=%d", + s.iteration, tool_ver, s.completeness, s.convention_compliance, + s.build_success, s.genuinely_wrong, s.valuable_extras, ) total_improve_cost = sum(imp.improvement_cost_usd for imp in improvements) total_gen_cost = sum(g.cost_usd for g in gen_results) @@ -184,8 +185,8 @@ async def cmd_run(args: argparse.Namespace) -> None: for r in results: ep_num = r.ep_url.rstrip("/").split("/")[-1] logger.info( - " EP #%s: completeness=%.1f%% precision=%.1f%%", - ep_num, r.median_completeness, r.median_precision, + " EP #%s: completeness=%.1f%%", + ep_num, r.median_completeness, ) logger.info("=" * 60) @@ -209,12 +210,13 @@ async def cmd_report(args: argparse.Namespace) -> None: IterationScore( iteration=s["iteration"], completeness=s["completeness"], - precision=s["precision"], convention_compliance=s["convention_compliance"], build_success=s["build_success"], - file_true_positives=s.get("file_tp", 0), - file_false_negatives=s.get("file_fn", 0), - file_false_positives=s.get("file_fp", 0), + files_matched=s.get("files_matched", 0), + files_missed=s.get("files_missed", 0), + genuinely_wrong=s.get("genuinely_wrong", 0), + valuable_extras=s.get("valuable_extras", 0), + auto_generated=s.get("auto_generated", 0), ) for s in data.get("iteration_scores", []) ] @@ -225,7 +227,6 @@ async def cmd_report(args: argparse.Namespace) -> None: implementation_prs=data.get("implementation_prs", []), iteration_scores=scores, median_completeness=data.get("median_completeness", 0), - median_precision=data.get("median_precision", 0), best_iteration=data.get("best_iteration", 0), score_variance=data.get("score_variance", {}), )) diff --git a/benchmark/compare.py b/benchmark/compare.py index 7e1187b..0406e92 100644 --- a/benchmark/compare.py +++ b/benchmark/compare.py @@ -337,7 +337,6 @@ def compare_iteration( tp, fn, fp = _compute_file_metrics(go_gen_files, go_truth_files) completeness_scores: list[float] = [] - precision_scores: list[float] = [] convention_scores: list[float] = [] all_outperformance: list[OutperformanceFinding] = [] @@ -363,15 +362,6 @@ def compare_iteration( + (overlap.get("func_match_rate", 0) * 0.2) ) - truth_struct_names = {s["name"] for s in truth_elements.get("structs", [])} - gen_struct_names = {s["name"] for s in gen_elements.get("structs", [])} - if gen_struct_names: - precision_scores.append( - len(gen_struct_names & truth_struct_names) / len(gen_struct_names) - ) - else: - precision_scores.append(0.0) - convention_scores.append(overlap.get("marker_match_rate", 0)) outperf = _detect_outperformance( @@ -401,64 +391,22 @@ def compare_iteration( ), )) - for fpath in go_gen_files - go_truth_files: - gen_path = gen_result.gen_dir / fpath - if gen_path.exists(): - gen_content = gen_path.read_text(errors="replace") - gen_elements = _extract_go_elements(gen_content) - if gen_elements.get("structs") or gen_elements.get("functions"): - precision_scores.append(0.3) - - genuinely_wrong_go = {f for f in classification.genuinely_wrong if f.endswith(".go")} - adjusted_precision_scores = list(precision_scores) - for fpath in go_gen_files - go_truth_files: - if fpath not in genuinely_wrong_go: - continue - gen_path = gen_result.gen_dir / fpath - if gen_path.exists(): - gen_content = gen_path.read_text(errors="replace") - gen_elements = _extract_go_elements(gen_content) - if gen_elements.get("structs") or gen_elements.get("functions"): - adjusted_precision_scores.append(0.3) - completeness = statistics.mean(completeness_scores) * 100 if completeness_scores else 0.0 - precision = statistics.mean(precision_scores) * 100 if precision_scores else 0.0 convention = statistics.mean(convention_scores) * 100 if convention_scores else 0.0 - num_penalizable = len(genuinely_wrong_go) - total_gen_go = len(go_gen_files) - if total_gen_go > 0: - non_penalized = total_gen_go - num_penalizable - if matched_files: - raw_match_rate = statistics.mean(precision_scores) if precision_scores else 0.0 - if non_penalized > 0: - adjusted_precision = ( - (raw_match_rate * len(matched_files) + 1.0 * len(go_gen_files - go_truth_files - genuinely_wrong_go)) - / (len(matched_files) + len(go_gen_files - go_truth_files - genuinely_wrong_go) + num_penalizable * 0.3) - ) if (len(matched_files) + len(go_gen_files - go_truth_files)) > 0 else raw_match_rate - else: - adjusted_precision = raw_match_rate - else: - adjusted_precision = 0.0 - adjusted_precision = adjusted_precision * 100 - else: - adjusted_precision = precision - if not matched_files and go_truth_files: completeness = 0.0 - precision = 0.0 - adjusted_precision = 0.0 score = IterationScore( iteration=gen_result.iteration, completeness=round(completeness, 1), - precision=round(precision, 1), - adjusted_precision=round(adjusted_precision, 1), convention_compliance=round(convention, 1), build_success=gen_result.build_success, - file_true_positives=tp, - file_false_negatives=fn, - file_false_positives=fp, + files_matched=tp, + files_missed=fn, + genuinely_wrong=len(classification.genuinely_wrong), + valuable_extras=len(classification.valuable_extra), + auto_generated=len(classification.auto_generated), file_classification=classification, ) @@ -491,14 +439,12 @@ def compare_all_iterations( dedup_findings.append(f) completeness_values = [s.completeness for s in all_scores] - precision_values = [s.precision for s in all_scores] score_variance = {} if len(completeness_values) > 1: score_variance["completeness"] = round(statistics.stdev(completeness_values), 2) - score_variance["precision"] = round(statistics.stdev(precision_values), 2) - best_iter = max(all_scores, key=lambda s: s.completeness + s.precision).iteration + best_iter = max(all_scores, key=lambda s: s.completeness - s.genuinely_wrong * 5).iteration stable: list[str] = [] unstable: list[str] = [] @@ -528,5 +474,4 @@ def compare_all_iterations( score_variance=score_variance, best_iteration=best_iter, median_completeness=round(statistics.median(completeness_values), 1), - median_precision=round(statistics.median(precision_values), 1), ) diff --git a/benchmark/config.yaml b/benchmark/config.yaml index d7e1917..866d208 100644 --- a/benchmark/config.yaml +++ b/benchmark/config.yaml @@ -1,8 +1,8 @@ benchmark_cases: - - ep_url: "https://github.com/openshift/enhancements/pull/1863" - repo_url: "https://github.com/openshift/zero-trust-workload-identity-manager" - description: "SPIRE federation support" - implementation_prs: [68, 82] + - ep_url: "https://github.com/openshift/enhancements/pull/1834" + repo_url: "https://github.com/openshift/external-secrets-operator" + description: "External Secrets Operator enhancement" + implementation_prs: [67, 74] settings: tools_to_benchmark: diff --git a/benchmark/models.py b/benchmark/models.py index 3f52857..8ee31dd 100644 --- a/benchmark/models.py +++ b/benchmark/models.py @@ -89,13 +89,13 @@ class FileClassification: class IterationScore: iteration: int completeness: float - precision: float # raw precision (all extras penalized) - adjusted_precision: float # excludes auto-generated and formatting-only extras convention_compliance: float build_success: bool - file_true_positives: int = 0 - file_false_negatives: int = 0 - file_false_positives: int = 0 + files_matched: int = 0 + files_missed: int = 0 + genuinely_wrong: int = 0 + valuable_extras: int = 0 + auto_generated: int = 0 file_classification: FileClassification = field(default_factory=FileClassification) @@ -112,7 +112,6 @@ class BenchmarkResult: score_variance: dict[str, float] = field(default_factory=dict) best_iteration: int = 0 median_completeness: float = 0.0 - median_precision: float = 0.0 tool_improvements: list[ToolImprovement] = field(default_factory=list) mode: str = "feedback_loop" # "feedback_loop" or "measurement" diff --git a/benchmark/report.py b/benchmark/report.py index 7648d4d..c3789d2 100644 --- a/benchmark/report.py +++ b/benchmark/report.py @@ -50,19 +50,27 @@ def generate_ep_report( lines.append("## Score Progression") lines.append("") - lines.append("| Iteration | Tool Version | Completeness | Raw Precision | Adjusted Precision | Convention | Build |") - lines.append("|-----------|-------------|-------------|--------------|-------------------|------------|-------|") + lines.append("| Iteration | Tool Version | Completeness | Convention | Build | Matched | Missed | Wrong | Extras | Auto |") + lines.append("|-----------|-------------|-------------|------------|-------|---------|--------|-------|--------|------|") for idx, s in enumerate(result.iteration_scores): build = "PASS" if s.build_success else "FAIL" tool_ver = gen_results[idx].tool_version if idx < len(gen_results) else "?" lines.append( - f"| {s.iteration} | {tool_ver} | {s.completeness:.1f}% | {s.precision:.1f}% | " - f"{s.adjusted_precision:.1f}% | {s.convention_compliance:.1f}% | {build} |" + f"| {s.iteration} | {tool_ver} | {s.completeness:.1f}% | " + f"{s.convention_compliance:.1f}% | {build} | {s.files_matched} | " + f"{s.files_missed} | {s.genuinely_wrong} | {s.valuable_extras} | {s.auto_generated} |" ) lines.append("") - lines.append("> **Raw Precision**: penalizes ALL extra files equally. " - "**Adjusted Precision**: excludes auto-generated artifacts and formatting-only changes, " - "and does not penalize valuable extra files (tests, validation, new types).") + lines.append("**Metric definitions:**") + lines.append("") + lines.append("- **Completeness**: What % of the human's structs, fields, and functions did the tool also generate? Higher = tool covered more of the ground truth.") + lines.append("- **Convention**: What % of kubebuilder markers (`+kubebuilder:validation:Required`, `+optional`, etc.) match between generated and ground truth?") + lines.append("- **Build**: Did `make build` pass on the generated code?") + lines.append("- **Matched**: Files the tool generated that also exist in the human implementation (true positives).") + lines.append("- **Missed**: Files in the human implementation that the tool did NOT generate (gaps to close).") + lines.append("- **Wrong**: Files the tool touched that it should not have -- unrelated to the EP (real errors).") + lines.append("- **Extras**: Files the tool generated that the human didn't -- tests, samples, better code organization (tool outperformed human).") + lines.append("- **Auto**: Files auto-generated by `make generate`/`make manifests` (expected build artifacts, not errors).") lines.append("") lines.append("### Summary") @@ -72,7 +80,6 @@ def generate_ep_report( first = result.iteration_scores[0] last = result.iteration_scores[-1] lines.append(f"- **Completeness**: {first.completeness:.1f}% -> {last.completeness:.1f}% ({last.completeness - first.completeness:+.1f}%)") - lines.append(f"- **Adjusted Precision**: {first.adjusted_precision:.1f}% -> {last.adjusted_precision:.1f}% ({last.adjusted_precision - first.adjusted_precision:+.1f}%)") total_gen_cost = sum(g.cost_usd for g in gen_results) total_imp_cost = sum(imp.improvement_cost_usd for imp in result.tool_improvements) lines.append(f"- **Total Cost**: ${total_gen_cost + total_imp_cost:.2f} (generation: ${total_gen_cost:.2f}, improvement: ${total_imp_cost:.2f})") @@ -82,20 +89,20 @@ def generate_ep_report( fc = s.file_classification if fc.auto_generated or fc.formatting_only or fc.valuable_extra or fc.genuinely_wrong: tool_ver = gen_results[idx].tool_version if idx < len(gen_results) else "?" - lines.append(f"### Iteration {s.iteration} ({tool_ver}) - Extra File Breakdown") + lines.append(f"### Iteration {s.iteration} ({tool_ver}) - Extra File Details") lines.append("") if fc.auto_generated: - lines.append(f"**Auto-generated artifacts** ({len(fc.auto_generated)} files) -- from `make generate`/`make manifests`, not tool errors:") + lines.append(f"**Auto-generated artifacts** ({len(fc.auto_generated)} files):") for f in fc.auto_generated: lines.append(f"- `{f}`") lines.append("") if fc.formatting_only: - lines.append(f"**Formatting-only changes** ({len(fc.formatting_only)} files) -- whitespace/import reordering, no behavior change:") + lines.append(f"**Formatting-only changes** ({len(fc.formatting_only)} files):") for f in fc.formatting_only: lines.append(f"- `{f}`") lines.append("") if fc.valuable_extra: - lines.append(f"**Valuable extras** ({len(fc.valuable_extra)} files) -- tool generated useful code the human didn't:") + lines.append(f"**Valuable extras** ({len(fc.valuable_extra)} files):") for f in fc.valuable_extra: lines.append(f"- `{f}`") lines.append("") @@ -214,20 +221,19 @@ def generate_ep_report( "description": result.description, "implementation_prs": result.implementation_prs, "median_completeness": result.median_completeness, - "median_precision": result.median_precision, "best_iteration": result.best_iteration, "score_variance": result.score_variance, "iteration_scores": [ { "iteration": s.iteration, "completeness": s.completeness, - "precision": s.precision, - "adjusted_precision": s.adjusted_precision, "convention_compliance": s.convention_compliance, "build_success": s.build_success, - "file_tp": s.file_true_positives, - "file_fn": s.file_false_negatives, - "file_fp": s.file_false_positives, + "files_matched": s.files_matched, + "files_missed": s.files_missed, + "genuinely_wrong": s.genuinely_wrong, + "valuable_extras": s.valuable_extras, + "auto_generated": s.auto_generated, "file_classification": { "auto_generated": s.file_classification.auto_generated, "formatting_only": s.file_classification.formatting_only, @@ -297,27 +303,27 @@ def generate_aggregate_report( lines.append("## Summary Table") lines.append("") - lines.append("| EP | Repo | Completeness | Precision | Best Iter | Consistency |") - lines.append("|----|------|-------------|-----------|-----------|-------------|") + lines.append("| EP | Repo | Completeness | Convention | Best Iter | Wrong | Extras |") + lines.append("|----|------|-------------|------------|-----------|-------|--------|") for r in results: ep_num = _ep_number(r.ep_url) repo = _repo_short_name(r.repo_url) - var = r.score_variance.get("completeness", 0) - consistency = "High" if var < 5 else ("Medium" if var < 15 else "Low") + best_score = next((s for s in r.iteration_scores if s.iteration == r.best_iteration), r.iteration_scores[0] if r.iteration_scores else None) + wrong = best_score.genuinely_wrong if best_score else 0 + extras = best_score.valuable_extras if best_score else 0 + conv = best_score.convention_compliance if best_score else 0 lines.append( f"| #{ep_num} | {repo} | {r.median_completeness:.1f}% | " - f"{r.median_precision:.1f}% | {r.best_iteration} | {consistency} |" + f"{conv:.1f}% | {r.best_iteration} | {wrong} | {extras} |" ) lines.append("") if len(results) > 1: all_completeness = [r.median_completeness for r in results] - all_precision = [r.median_precision for r in results] import statistics lines.append("### Averages Across All EPs") lines.append("") lines.append(f"- **Mean Completeness**: {statistics.mean(all_completeness):.1f}%") - lines.append(f"- **Mean Precision**: {statistics.mean(all_precision):.1f}%") lines.append("") total_outperf = sum(len(r.outperformance_findings) for r in results) diff --git a/benchmark/runner.py b/benchmark/runner.py index cfbe24d..3b44f44 100644 --- a/benchmark/runner.py +++ b/benchmark/runner.py @@ -173,11 +173,10 @@ def _build_improvement_prompt( ### Scores - Completeness: {score.completeness:.1f}% (what % of ground truth structs/fields/functions were generated) -- Raw Precision: {score.precision:.1f}% (penalizes ALL extra files) -- Adjusted Precision: {score.adjusted_precision:.1f}% (excludes auto-generated artifacts and valuable extras) - Convention Compliance: {score.convention_compliance:.1f}% - Build Success: {"PASS" if score.build_success else "FAIL"} -- Files: {score.file_true_positives} matched, {score.file_false_negatives} missed, {score.file_false_positives} extra +- Files matched: {score.files_matched} | Files missed: {score.files_missed} +- Genuinely wrong files: {score.genuinely_wrong} | Valuable extras: {score.valuable_extras} | Auto-generated: {score.auto_generated} ### Understanding Extra Files From 3481cf18b24502b1323a7366d50d5f741ba272d4 Mon Sep 17 00:00:00 2001 From: Raushan Singh Date: Wed, 6 May 2026 16:21:48 +0530 Subject: [PATCH 3/6] fix: exclude vendor/ and e2e test files from benchmark comparison vendor/ files are dependency artifacts, not implementation code. e2e tests are handled by a separate OAPE command (/oape:e2e-generate), so they should not be part of the api-generate + api-implement evaluation. Excluded paths: vendor/*, test/e2e/*, tests/e2e/*, e2e/*, *_e2e_test.go, *_e2e_suite_test.go Co-authored-by: Cursor --- benchmark/compare.py | 16 ++++++++++++++-- benchmark/ground_truth.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/benchmark/compare.py b/benchmark/compare.py index 0406e92..c00a9e2 100644 --- a/benchmark/compare.py +++ b/benchmark/compare.py @@ -323,13 +323,25 @@ def _detect_outperformance( return findings +def _should_exclude_from_comparison(filepath: str) -> bool: + """Exclude vendor and e2e test files from comparison.""" + if filepath.startswith("vendor/"): + return True + for pattern in ("test/e2e/", "tests/e2e/", "e2e/", "_e2e_test.go", "_e2e_suite_test.go"): + if pattern in filepath: + return True + return False + + def compare_iteration( gen_result: GenerationResult, truth: GroundTruth, ) -> tuple[IterationScore, list[OutperformanceFinding]]: """Compare a single iteration's output against ground truth.""" - gen_all_files = set(gen_result.files_created + gen_result.files_modified) - truth_all_files = set(truth.files_added + truth.files_modified) + gen_all_files = {f for f in set(gen_result.files_created + gen_result.files_modified) + if not _should_exclude_from_comparison(f)} + truth_all_files = {f for f in set(truth.files_added + truth.files_modified) + if not _should_exclude_from_comparison(f)} go_truth_files = {f for f in truth_all_files if f.endswith(".go")} go_gen_files = {f for f in gen_all_files if f.endswith(".go")} diff --git a/benchmark/ground_truth.py b/benchmark/ground_truth.py index c7c18d0..84248c6 100644 --- a/benchmark/ground_truth.py +++ b/benchmark/ground_truth.py @@ -14,6 +14,29 @@ logger = logging.getLogger(__name__) +EXCLUDED_PATH_PREFIXES = [ + "vendor/", + "test/e2e/", + "tests/e2e/", + "e2e/", +] + +EXCLUDED_PATH_PATTERNS = [ + "_e2e_test.go", + "_e2e_suite_test.go", +] + + +def _should_exclude(filepath: str) -> bool: + """Check if a file should be excluded from ground truth comparison.""" + for prefix in EXCLUDED_PATH_PREFIXES: + if filepath.startswith(prefix): + return True + for pattern in EXCLUDED_PATH_PATTERNS: + if pattern in filepath: + return True + return False + def _run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: logger.debug("Running: %s", " ".join(cmd)) @@ -49,6 +72,7 @@ def extract_combined_truth(repo_url: str, timeline: PRTimeline) -> GroundTruth: files_added: list[str] = [] files_modified: list[str] = [] + excluded_count = 0 for line in stat_result.stdout.strip().split("\n"): if not line.strip(): @@ -57,11 +81,17 @@ def extract_combined_truth(repo_url: str, timeline: PRTimeline) -> GroundTruth: if len(parts) < 2: continue status, filepath = parts[0].strip(), parts[1].strip() + if _should_exclude(filepath): + excluded_count += 1 + continue if status == "A": files_added.append(filepath) elif status in ("M", "R", "C"): files_modified.append(filepath) + if excluded_count: + logger.info("Excluded %d files (vendor/, e2e tests) from ground truth", excluded_count) + _run(["git", "checkout", "--quiet", final], cwd=truth_dir) diff_hunks: dict[str, str] = {} From a84c04c74d74d33f3a08e30d6d528c911d9bafeb Mon Sep 17 00:00:00 2001 From: Raushan Singh Date: Wed, 6 May 2026 16:28:54 +0530 Subject: [PATCH 4/6] refactor: replace per-EP feedback loop with measure-improve-verify phases Instead of improving the tool after every iteration of every EP (which causes overfitting and bloat), use a three-phase approach: Phase 1 (measure): Run all EPs once with the original tool Phase 2 (improve): Analyze ALL results together, find cross-EP patterns, make ONE set of concise improvements Phase 3 (verify): Run all EPs again with the improved tool This produces targeted, general improvements that work across all EPs instead of EP-specific bloat. CLI: benchmark.py measure|improve|verify|full Co-authored-by: Cursor --- benchmark/benchmark.py | 425 ++++++++++++++++++++++++----------------- benchmark/runner.py | 151 +++++++++++++++ 2 files changed, 404 insertions(+), 172 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 6780695..0ba25d1 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -1,8 +1,12 @@ #!/usr/bin/env python3 """OAPE Benchmark Pipeline CLI. -Runs OAPE tools against curated EP-to-implementation mappings, -compares generated output vs real merged code, and reports quality. +Two-phase approach: + Phase 1 (measure): Run all EPs with original tool, collect baselines + Phase 2 (improve): Analyze all results, make ONE set of concise improvements + Phase 3 (verify): Run all EPs again with improved tool to confirm improvements + +Or use `full` to run all three phases in sequence. """ import argparse @@ -16,12 +20,12 @@ import yaml -from compare import compare_all_iterations +from compare import compare_all_iterations, compare_iteration from ground_truth import extract_combined_truth from isolate import prepare_generation_env, resolve_pr_timeline -from models import BenchmarkCase, BenchmarkConfig, BenchmarkResult +from models import BenchmarkCase, BenchmarkConfig, BenchmarkResult, IterationScore from report import generate_aggregate_report, generate_ep_report -from runner import run_feedback_loop, PLUGIN_DIR +from runner import run_single_iteration, improve_tool_from_all_results, PLUGIN_DIR, TOOL_FILES logging.basicConfig( level=logging.INFO, @@ -31,7 +35,7 @@ logger = logging.getLogger("benchmark") -def load_config(config_path: str, iterations_override: int | None = None) -> BenchmarkConfig: +def load_config(config_path: str) -> BenchmarkConfig: with open(config_path) as f: raw = yaml.safe_load(f) @@ -45,12 +49,11 @@ def load_config(config_path: str, iterations_override: int | None = None) -> Ben )) settings = raw.get("settings", {}) - iterations = iterations_override or settings.get("iterations", 3) return BenchmarkConfig( cases=cases, tools_to_benchmark=settings.get("tools_to_benchmark", ["api-generate", "api-implement"]), - iterations=iterations, + iterations=settings.get("iterations", 1), output_dir=settings.get("output_dir", "benchmark/results"), parallel=settings.get("parallel", False), model=settings.get("model", "claude-opus-4-6"), @@ -58,15 +61,24 @@ def load_config(config_path: str, iterations_override: int | None = None) -> Ben ) -async def run_single_case( +def _ep_num(ep_url: str) -> str: + return ep_url.rstrip("/").split("/")[-1] + + +def _repo_name(repo_url: str) -> str: + return repo_url.rstrip("/").split("/")[-1].removesuffix(".git") + + +async def run_single_ep( case: BenchmarkCase, config: BenchmarkConfig, output_dir: Path, + phase_label: str = "measure", force: bool = False, ) -> BenchmarkResult | None: - """Run the benchmark pipeline for a single EP case.""" - ep_num = case.ep_url.rstrip("/").split("/")[-1] - repo_name = case.repo_url.rstrip("/").split("/")[-1].removesuffix(".git") + """Run the OAPE tool once on a single EP and score it.""" + ep_num = _ep_num(case.ep_url) + repo_name = _repo_name(case.repo_url) result_dir = output_dir / repo_name / f"ep-{ep_num}" if result_dir.exists() and not force: @@ -76,14 +88,13 @@ async def run_single_case( return None logger.info("=" * 60) - logger.info("FEEDBACK LOOP BENCHMARK: EP #%s - %s", ep_num, case.description) + logger.info("[%s] EP #%s - %s", phase_label.upper(), ep_num, case.description) logger.info("Repo: %s", case.repo_url) logger.info("Implementation PRs: %s", case.implementation_prs) - logger.info("Iterations: %d (with tool improvement between each)", config.iterations) logger.info("Model: %s | Effort: %s", config.model, config.effort) logger.info("=" * 60) - logger.info("Step 1: Resolving PR timeline...") + logger.info("Resolving PR timeline...") timeline = resolve_pr_timeline(case.repo_url, case.implementation_prs) logger.info( "Timeline: baseline=%s, final=%s, %d files changed", @@ -92,160 +103,206 @@ async def run_single_case( len(timeline.all_changed_files), ) - logger.info("Step 2: Extracting ground truth...") + logger.info("Extracting ground truth...") truth = extract_combined_truth(case.repo_url, timeline) logger.info( "Ground truth: %d added, %d modified files", len(truth.files_added), len(truth.files_modified), ) - logger.info("Step 3: Preparing isolated generation environment...") + logger.info("Preparing isolated environment...") source_env = prepare_generation_env(case.repo_url, timeline) - if source_env.warnings: - for w in source_env.warnings: - logger.warning(w) - backup_dir = output_dir / repo_name / f"ep-{ep_num}" / "tool-backups" - backup_dir.mkdir(parents=True, exist_ok=True) - - logger.info("Step 4: Running feedback loop (%d iterations)...", config.iterations) - gen_results, improvements = await run_feedback_loop( + logger.info("Running OAPE tools...") + gen_result = await run_single_iteration( ep_url=case.ep_url, source_env=source_env, - truth=truth, - num_iterations=config.iterations, + iteration=1, plugin_dir=PLUGIN_DIR, - backup_dir=backup_dir, model=config.model, effort=config.effort, ) - logger.info("Step 5: Final comparison across all iterations...") + score, outperf = compare_iteration(gen_result, truth) + logger.info( + "Results: completeness=%.1f%% convention=%.1f%% build=%s matched=%d missed=%d wrong=%d extras=%d", + score.completeness, score.convention_compliance, score.build_success, + score.files_matched, score.files_missed, score.genuinely_wrong, score.valuable_extras, + ) + benchmark_result = compare_all_iterations( - gen_results=gen_results, + gen_results=[gen_result], truth=truth, ep_url=case.ep_url, repo_url=case.repo_url, description=case.description, implementation_prs=case.implementation_prs, ) - benchmark_result.tool_improvements = improvements - benchmark_result.mode = "feedback_loop" + benchmark_result.mode = phase_label - logger.info("Step 6: Generating report...") report_path = generate_ep_report( result=benchmark_result, - gen_results=gen_results, + gen_results=[gen_result], truth=truth, output_dir=output_dir, ) - logger.info("Report: %s", report_path) - logger.info("=" * 40) - logger.info("SCORE PROGRESSION:") - for s in benchmark_result.iteration_scores: - tool_ver = gen_results[s.iteration - 1].tool_version if s.iteration <= len(gen_results) else "?" - logger.info( - " Iter %d (%s): completeness=%.1f%% convention=%.1f%% build=%s wrong=%d extras=%d", - s.iteration, tool_ver, s.completeness, s.convention_compliance, - s.build_success, s.genuinely_wrong, s.valuable_extras, - ) - total_improve_cost = sum(imp.improvement_cost_usd for imp in improvements) - total_gen_cost = sum(g.cost_usd for g in gen_results) - logger.info("Total cost: generation=$%.2f + improvement=$%.2f = $%.2f", - total_gen_cost, total_improve_cost, total_gen_cost + total_improve_cost) - logger.info("=" * 40) - - if benchmark_result.outperformance_findings: - logger.info( - "Tool outperformed human in %d cases", - len(benchmark_result.outperformance_findings), - ) shutil.rmtree(source_env.path, ignore_errors=True) - return benchmark_result -async def cmd_run(args: argparse.Namespace) -> None: - config = load_config(args.config, args.iterations) - output_dir = Path(config.output_dir) +async def cmd_measure(args: argparse.Namespace) -> None: + """Phase 1: Run all EPs with original tool.""" + config = load_config(args.config) + output_dir = Path(config.output_dir) / "phase1-measure" output_dir.mkdir(parents=True, exist_ok=True) results: list[BenchmarkResult] = [] - - for case in config.cases: - result = await run_single_case(case, config, output_dir, force=args.force) + for i, case in enumerate(config.cases, 1): + logger.info("\n>>> EP %d of %d <<<", i, len(config.cases)) + result = await run_single_ep(case, config, output_dir, "measure", force=args.force) if result: results.append(result) if results: logger.info("\n" + "=" * 60) - logger.info("ALL BENCHMARKS COMPLETE") + logger.info("PHASE 1 COMPLETE: %d EPs measured", len(results)) for r in results: - ep_num = r.ep_url.rstrip("/").split("/")[-1] + best = r.iteration_scores[0] if r.iteration_scores else None logger.info( - " EP #%s: completeness=%.1f%%", - ep_num, r.median_completeness, + " EP #%s: completeness=%.1f%% wrong=%d extras=%d", + _ep_num(r.ep_url), r.median_completeness, + best.genuinely_wrong if best else 0, best.valuable_extras if best else 0, ) logger.info("=" * 60) if len(results) > 1: - agg_path = generate_aggregate_report(results, output_dir) - logger.info("Aggregate report: %s", agg_path) + generate_aggregate_report(results, output_dir) -async def cmd_report(args: argparse.Namespace) -> None: - results_dir = Path(args.results_dir) - if not results_dir.exists(): - logger.error("Results directory not found: %s", results_dir) +async def cmd_improve(args: argparse.Namespace) -> None: + """Phase 2: Analyze all results and make ONE set of concise improvements.""" + config = load_config(args.config) + measure_dir = Path(config.output_dir) / "phase1-measure" + + if not measure_dir.exists(): + logger.error("No phase1-measure results found. Run `measure` first.") sys.exit(1) - results: list[BenchmarkResult] = [] - for report_json in results_dir.rglob("report.json"): + results_data: list[dict] = [] + for report_json in measure_dir.rglob("report.json"): with open(report_json) as f: - data = json.load(f) - from models import IterationScore - scores = [ - IterationScore( - iteration=s["iteration"], - completeness=s["completeness"], - convention_compliance=s["convention_compliance"], - build_success=s["build_success"], - files_matched=s.get("files_matched", 0), - files_missed=s.get("files_missed", 0), - genuinely_wrong=s.get("genuinely_wrong", 0), - valuable_extras=s.get("valuable_extras", 0), - auto_generated=s.get("auto_generated", 0), - ) - for s in data.get("iteration_scores", []) - ] - results.append(BenchmarkResult( - ep_url=data["ep_url"], - repo_url=data["repo_url"], - description=data.get("description", ""), - implementation_prs=data.get("implementation_prs", []), - iteration_scores=scores, - median_completeness=data.get("median_completeness", 0), - best_iteration=data.get("best_iteration", 0), - score_variance=data.get("score_variance", {}), - )) + results_data.append(json.load(f)) - if not results: - logger.error("No benchmark results found in %s", results_dir) + if not results_data: + logger.error("No report.json files found in %s", measure_dir) sys.exit(1) - agg_path = generate_aggregate_report(results, results_dir) - logger.info("Aggregate report: %s", agg_path) + truth_data: dict[str, dict] = {} + for rd in results_data: + ep_num = _ep_num(rd["ep_url"]) + repo_name = _repo_name(rd["repo_url"]) + truth_dir = measure_dir / repo_name / f"ep-{ep_num}" / "truth" + if truth_dir.exists(): + combined_diff = (truth_dir / "combined.diff").read_text() if (truth_dir / "combined.diff").exists() else "" + truth_data[ep_num] = {"diff_preview": combined_diff[:5000]} + + iter_diffs: dict[str, str] = {} + for rd in results_data: + ep_num = _ep_num(rd["ep_url"]) + repo_name = _repo_name(rd["repo_url"]) + iter_dir = measure_dir / repo_name / f"ep-{ep_num}" / "iter-1" + if iter_dir and (iter_dir / "diff.patch").exists(): + iter_diffs[ep_num] = (iter_dir / "diff.patch").read_text()[:5000] + + backup_dir = Path(config.output_dir) / "tool-backups" + backup_dir.mkdir(parents=True, exist_ok=True) + + logger.info("Analyzing %d EP results and improving tool...", len(results_data)) + improvement = await improve_tool_from_all_results( + results_data=results_data, + truth_data=truth_data, + iter_diffs=iter_diffs, + plugin_dir=PLUGIN_DIR, + backup_dir=backup_dir, + model=config.model, + effort=config.effort, + ) + + if improvement.files_changed: + logger.info("Tool improved: %d files changed", len(improvement.files_changed)) + for fname, diff in improvement.files_changed.items(): + added = sum(1 for l in diff.split("\n") if l.startswith("+") and not l.startswith("+++")) + removed = sum(1 for l in diff.split("\n") if l.startswith("-") and not l.startswith("---")) + logger.info(" %s: +%d/-%d lines", fname, added, removed) + logger.info("Cost: $%.2f", improvement.improvement_cost_usd) + else: + logger.info("No changes made to tool files.") + + +async def cmd_verify(args: argparse.Namespace) -> None: + """Phase 3: Run all EPs again with improved tool.""" + config = load_config(args.config) + output_dir = Path(config.output_dir) / "phase3-verify" + output_dir.mkdir(parents=True, exist_ok=True) + + results: list[BenchmarkResult] = [] + for i, case in enumerate(config.cases, 1): + logger.info("\n>>> EP %d of %d <<<", i, len(config.cases)) + result = await run_single_ep(case, config, output_dir, "verify", force=args.force) + if result: + results.append(result) + + if results: + logger.info("\n" + "=" * 60) + logger.info("PHASE 3 COMPLETE: %d EPs verified with improved tool", len(results)) + for r in results: + best = r.iteration_scores[0] if r.iteration_scores else None + logger.info( + " EP #%s: completeness=%.1f%% wrong=%d extras=%d", + _ep_num(r.ep_url), r.median_completeness, + best.genuinely_wrong if best else 0, best.valuable_extras if best else 0, + ) + logger.info("=" * 60) + + if len(results) > 1: + generate_aggregate_report(results, output_dir) + + +async def cmd_full(args: argparse.Namespace) -> None: + """Run all three phases in sequence.""" + logger.info("=== PHASE 1: MEASURE (original tool) ===") + await cmd_measure(args) + + logger.info("\n=== PHASE 2: IMPROVE (analyze and edit tool) ===") + await cmd_improve(args) + + logger.info("\n=== PHASE 3: VERIFY (improved tool) ===") + await cmd_verify(args) + + config = load_config(args.config) + logger.info("\n=== FINAL: Restoring original tool files ===") + backup_dir = Path(config.output_dir) / "tool-backups" / "original" + if backup_dir.exists(): + for rel in TOOL_FILES: + src = backup_dir / rel + dst = Path(PLUGIN_DIR) / rel + if src.exists(): + shutil.copy2(src, dst) + logger.info("Original tool files restored. Improved versions saved in tool-backups/.") def cmd_push(args: argparse.Namespace) -> None: results_dir = Path(args.results_dir or "benchmark/results") repo_name = args.repo.rstrip("/").split("/")[-1].removesuffix(".git") - iter_dir = results_dir / repo_name / f"ep-{args.ep}" / f"iter-{args.iteration}" - if not iter_dir.exists(): - logger.error("Iteration directory not found: %s", iter_dir) + for phase in ["phase3-verify", "phase1-measure"]: + iter_dir = results_dir / phase / repo_name / f"ep-{args.ep}" / "iter-1" + if iter_dir.exists(): + break + else: + logger.error("No results found for EP #%s in %s", args.ep, results_dir) sys.exit(1) patch_file = iter_dir / "diff.patch" @@ -253,27 +310,21 @@ def cmd_push(args: argparse.Namespace) -> None: logger.error("No diff.patch found or patch is empty in %s", iter_dir) sys.exit(1) - gen_dir = iter_dir / "generated" - import tempfile work_dir = Path(tempfile.mkdtemp(prefix="oape-bench-push-")) logger.info("Cloning %s for PR creation...", args.repo) + subprocess.run(["git", "clone", "--quiet", args.repo, str(work_dir)], check=True) subprocess.run( - ["git", "clone", "--quiet", args.repo, str(work_dir)], - check=True, - ) - subprocess.run( - ["git", "checkout", "-b", f"oape-benchmark/ep-{args.ep}-iter{args.iteration}"], + ["git", "checkout", "-b", f"oape-benchmark/ep-{args.ep}"], cwd=work_dir, check=True, ) - logger.info("Applying patch from iteration %d...", args.iteration) + logger.info("Applying patch...") patch_content = patch_file.read_text() result = subprocess.run( ["git", "apply", "--verbose", "-"], - input=patch_content, text=True, - cwd=work_dir, capture_output=True, + input=patch_content, text=True, cwd=work_dir, capture_output=True, ) if result.returncode != 0: logger.error("Failed to apply patch: %s", result.stderr) @@ -281,13 +332,13 @@ def cmd_push(args: argparse.Namespace) -> None: subprocess.run(["git", "add", "-A"], cwd=work_dir, check=True) subprocess.run( - ["git", "commit", "-m", args.title or f"feat: OAPE benchmark EP-{args.ep} iteration {args.iteration}"], + ["git", "commit", "-m", args.title or f"feat: OAPE generated implementation for EP-{args.ep}"], cwd=work_dir, check=True, ) logger.info("Pushing branch...") subprocess.run( - ["git", "push", "-u", "origin", f"oape-benchmark/ep-{args.ep}-iter{args.iteration}"], + ["git", "push", "-u", "origin", f"oape-benchmark/ep-{args.ep}"], cwd=work_dir, check=True, ) @@ -296,8 +347,8 @@ def cmd_push(args: argparse.Namespace) -> None: ["gh", "pr", "create", "--repo", args.repo, "--base", args.base_branch, - "--title", args.title or f"feat: OAPE benchmark EP-{args.ep} iteration {args.iteration}", - "--body", f"Generated by OAPE benchmark pipeline (EP #{args.ep}, iteration {args.iteration})"], + "--title", args.title or f"feat: OAPE generated implementation for EP-{args.ep}", + "--body", f"Generated by OAPE benchmark pipeline (EP #{args.ep})"], cwd=work_dir, capture_output=True, text=True, ) if pr_result.returncode == 0: @@ -312,68 +363,98 @@ def main() -> None: parser = argparse.ArgumentParser( description="OAPE Benchmark Pipeline", formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Workflow: + 1. measure - Run all EPs with original tool, collect baselines + 2. improve - Analyze all results, make ONE set of concise improvements + 3. verify - Run all EPs again with improved tool + Or: full - Run all three phases in sequence + """, ) subparsers = parser.add_subparsers(dest="command", required=True) - run_parser = subparsers.add_parser("run", help="Run benchmark cases") - run_parser.add_argument("--config", default="benchmark/config.yaml", help="Config YAML file") - run_parser.add_argument("--iterations", type=int, help="Override iteration count") - run_parser.add_argument("--force", action="store_true", help="Re-run completed cases") - run_parser.add_argument("--ep-url", help="Single EP URL (ad-hoc mode)") - run_parser.add_argument("--repo", help="Repo URL (ad-hoc mode)") - run_parser.add_argument("--prs", help="Comma-separated PR numbers (ad-hoc mode)") - - report_parser = subparsers.add_parser("report", help="Generate aggregate report") - report_parser.add_argument("--results-dir", default="benchmark/results", help="Results directory") - - full_parser = subparsers.add_parser("full", help="Run benchmarks + generate aggregate report") - full_parser.add_argument("--config", default="benchmark/config.yaml", help="Config YAML file") - full_parser.add_argument("--iterations", type=int, help="Override iteration count") - full_parser.add_argument("--force", action="store_true", help="Re-run completed cases") - - push_parser = subparsers.add_parser("push", help="Push an iteration as a PR") - push_parser.add_argument("--ep", required=True, help="EP number") - push_parser.add_argument("--repo", required=True, help="Repo URL") - push_parser.add_argument("--iteration", type=int, required=True, help="Iteration number") - push_parser.add_argument("--base-branch", default="main", help="Base branch for PR") - push_parser.add_argument("--title", help="PR title") - push_parser.add_argument("--results-dir", help="Results directory") + measure_p = subparsers.add_parser("measure", help="Phase 1: Run all EPs with original tool") + measure_p.add_argument("--config", default="benchmark/config.yaml") + measure_p.add_argument("--force", action="store_true") - args = parser.parse_args() + improve_p = subparsers.add_parser("improve", help="Phase 2: Analyze results, improve tool once") + improve_p.add_argument("--config", default="benchmark/config.yaml") - if args.command == "run": - if args.ep_url and args.repo and args.prs: - prs = [int(p.strip()) for p in args.prs.split(",")] - config_dict = { - "benchmark_cases": [{ - "ep_url": args.ep_url, - "repo_url": args.repo, - "description": "Ad-hoc benchmark", - "implementation_prs": prs, - }], - "settings": { - "iterations": args.iterations or 3, - "output_dir": "benchmark/results", - }, - } - import tempfile - tmp = Path(tempfile.mktemp(suffix=".yaml")) - tmp.write_text(yaml.dump(config_dict)) - args.config = str(tmp) - - asyncio.run(cmd_run(args)) + verify_p = subparsers.add_parser("verify", help="Phase 3: Run all EPs with improved tool") + verify_p.add_argument("--config", default="benchmark/config.yaml") + verify_p.add_argument("--force", action="store_true") - elif args.command == "report": - asyncio.run(cmd_report(args)) + full_p = subparsers.add_parser("full", help="Run all three phases") + full_p.add_argument("--config", default="benchmark/config.yaml") + full_p.add_argument("--force", action="store_true") - elif args.command == "full": - asyncio.run(cmd_run(args)) - args_report = argparse.Namespace(results_dir=load_config(args.config).output_dir) - asyncio.run(cmd_report(args_report)) + report_p = subparsers.add_parser("report", help="Generate aggregate report from results") + report_p.add_argument("--results-dir", default="benchmark/results") + + push_p = subparsers.add_parser("push", help="Push results as a PR") + push_p.add_argument("--ep", required=True) + push_p.add_argument("--repo", required=True) + push_p.add_argument("--base-branch", default="main") + push_p.add_argument("--title") + push_p.add_argument("--results-dir") + args = parser.parse_args() + + if args.command == "measure": + asyncio.run(cmd_measure(args)) + elif args.command == "improve": + asyncio.run(cmd_improve(args)) + elif args.command == "verify": + asyncio.run(cmd_verify(args)) + elif args.command == "full": + asyncio.run(cmd_full(args)) + elif args.command == "report": + asyncio.run(cmd_report(args)) elif args.command == "push": cmd_push(args) +async def cmd_report(args: argparse.Namespace) -> None: + results_dir = Path(args.results_dir) + if not results_dir.exists(): + logger.error("Results directory not found: %s", results_dir) + sys.exit(1) + + results: list[BenchmarkResult] = [] + for report_json in results_dir.rglob("report.json"): + with open(report_json) as f: + data = json.load(f) + scores = [ + IterationScore( + iteration=s["iteration"], + completeness=s["completeness"], + convention_compliance=s["convention_compliance"], + build_success=s["build_success"], + files_matched=s.get("files_matched", 0), + files_missed=s.get("files_missed", 0), + genuinely_wrong=s.get("genuinely_wrong", 0), + valuable_extras=s.get("valuable_extras", 0), + auto_generated=s.get("auto_generated", 0), + ) + for s in data.get("iteration_scores", []) + ] + results.append(BenchmarkResult( + ep_url=data["ep_url"], + repo_url=data["repo_url"], + description=data.get("description", ""), + implementation_prs=data.get("implementation_prs", []), + iteration_scores=scores, + median_completeness=data.get("median_completeness", 0), + best_iteration=data.get("best_iteration", 0), + score_variance=data.get("score_variance", {}), + )) + + if not results: + logger.error("No results found in %s", results_dir) + sys.exit(1) + + generate_aggregate_report(results, results_dir) + + if __name__ == "__main__": main() diff --git a/benchmark/runner.py b/benchmark/runner.py index 3b44f44..918312f 100644 --- a/benchmark/runner.py +++ b/benchmark/runner.py @@ -472,6 +472,157 @@ async def improve_tool( ) +async def improve_tool_from_all_results( + results_data: list[dict], + truth_data: dict[str, dict], + iter_diffs: dict[str, str], + plugin_dir: str = PLUGIN_DIR, + backup_dir: Path | None = None, + model: str = "claude-opus-4-6", + effort: str = "max", +) -> ToolImprovement: + """Analyze results from ALL EPs and make ONE set of concise improvements. + + This is the Phase 2 function -- it looks at patterns across all EPs + and makes generic improvements, not EP-specific ones. + """ + logger.info("=== Analyzing %d EP results for cross-EP patterns ===", len(results_data)) + + if backup_dir: + _backup_tool_files(plugin_dir, backup_dir, "original") + + tool_contents: dict[str, str] = {} + for rel in TOOL_FILES: + path = Path(plugin_dir) / rel + if path.exists(): + tool_contents[rel] = path.read_text() + + ep_summaries = [] + for rd in results_data: + ep_num = rd["ep_url"].rstrip("/").split("/")[-1] + scores = rd.get("iteration_scores", [{}]) + s = scores[0] if scores else {} + fc = s.get("file_classification", {}) + + truth_diff = truth_data.get(ep_num, {}).get("diff_preview", "") + gen_diff = iter_diffs.get(ep_num, "") + + ep_summaries.append(f""" +### EP #{ep_num}: {rd.get('description', '')} +- Repo: {rd.get('repo_url', '')} +- Completeness: {s.get('completeness', 0):.1f}% +- Convention: {s.get('convention_compliance', 0):.1f}% +- Build: {"PASS" if s.get('build_success') else "FAIL"} +- Matched: {s.get('files_matched', 0)} | Missed: {s.get('files_missed', 0)} | Wrong: {s.get('genuinely_wrong', 0)} | Extras: {s.get('valuable_extras', 0)} +- Genuinely wrong files: {fc.get('genuinely_wrong', [])} +- Missed ground truth files (first 15): {(rd.get('implementation_prs', []))} + +Ground truth diff (first 5000 chars): +```diff +{truth_diff} +``` + +Generated diff (first 5000 chars): +```diff +{gen_diff} +``` +""") + + tool_sections = [] + for rel, content in tool_contents.items(): + tool_sections.append(f"### FILE: {rel}\n```markdown\n{content}\n```") + + prompt = f"""You are improving OAPE tool instructions based on benchmark results from {len(results_data)} different Enhancement Proposals across different OpenShift operators. + +## CRITICAL RULES + +1. Make GENERIC improvements only. Do NOT reference specific EP numbers, feature names, struct names, or repo names. +2. Keep edits CONCISE. Add short rules or checklist items, not paragraphs of explanation. +3. Each edit must address a pattern seen across MULTIPLE EPs, not just one. +4. Do NOT bloat the files. If a 3-line rule can replace a 20-line explanation, use the 3-line rule. +5. Preserve the existing file structure and phases. + +## Results from {len(results_data)} EPs + +{''.join(ep_summaries)} + +## Current Tool Instructions + +{chr(10).join(tool_sections)} + +## What to do + +Look at the patterns across ALL EPs: +- What types of files are consistently missed? Add a concise rule. +- What types of files are consistently marked "genuinely wrong"? Add a concise "do not" rule. +- Are there common convention gaps? Add a short checklist item. +- Are there controller patterns the tool misses? Add a brief pattern. + +Make your edits to the tool files. Keep them short and to the point. +After editing, list what you changed in 2-3 bullet points. +""" + + improvement_log: list[str] = [] + cost_usd = 0.0 + + try: + from claude_agent_sdk import ( + query, + ClaudeAgentOptions, + AssistantMessage, + ResultMessage, + TextBlock, + ToolUseBlock, + ) + + options = ClaudeAgentOptions( + model=model, + effort=effort, + system_prompt=( + "You improve AI code generation tools by making concise, targeted edits. " + "You never add bloat. Every edit is a short rule or checklist item. " + "You never reference specific EPs, features, or repos." + ), + cwd=plugin_dir, + permission_mode="bypassPermissions", + allowed_tools=[ + "Read", "Write", "Edit", "MultiEdit", + "Bash", "Glob", "Grep", + ], + ) + + async for message in query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + improvement_log.append(block.text) + elif isinstance(block, ToolUseBlock): + improvement_log.append(f"[tool:{block.name}]") + elif isinstance(message, ResultMessage): + cost_usd = message.total_cost_usd + if message.result: + improvement_log.append(message.result) + + except ImportError: + logger.warning("claude_agent_sdk not available") + improvement_log.append("[DRY RUN]") + except Exception: + improvement_log.append(f"[ERROR] {traceback.format_exc()}") + logger.error("Tool improvement failed: %s", traceback.format_exc()) + + diffs = {} + if backup_dir: + diffs = _diff_tool_files(plugin_dir, backup_dir, "original") + _backup_tool_files(plugin_dir, backup_dir, "improved") + + return ToolImprovement( + iteration=0, + files_changed=diffs, + analysis_summary="\n".join(improvement_log[-5:]) if improvement_log else "", + improvement_cost_usd=cost_usd, + ) + + async def run_feedback_loop( ep_url: str, source_env: IsolatedEnv, From 3e63567e461b52d46fc881cc7de0c7e6b02950b2 Mon Sep 17 00:00:00 2001 From: Raushan Singh Date: Thu, 7 May 2026 11:17:52 +0530 Subject: [PATCH 5/6] chore: cleanup dead code, add 9 EP benchmark config - Remove unused _build_improvement_prompt, improve_tool, run_feedback_loop - Fix mislabeled "missed files" line in improvement prompt - Remove unused imports - Add config.yaml with 9 EPs across 4 operators - EP #1964 excluded (openshift/must-gather is Shell, not Go) Co-authored-by: Cursor --- benchmark/benchmark.py | 22 ++- benchmark/config.yaml | 51 ++++++- benchmark/isolate.py | 15 +- benchmark/runner.py | 306 +---------------------------------------- 4 files changed, 79 insertions(+), 315 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 0ba25d1..1ad8724 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -161,9 +161,14 @@ async def cmd_measure(args: argparse.Namespace) -> None: results: list[BenchmarkResult] = [] for i, case in enumerate(config.cases, 1): logger.info("\n>>> EP %d of %d <<<", i, len(config.cases)) - result = await run_single_ep(case, config, output_dir, "measure", force=args.force) - if result: - results.append(result) + try: + result = await run_single_ep(case, config, output_dir, "measure", force=args.force) + if result: + results.append(result) + except Exception: + ep_num = _ep_num(case.ep_url) + logger.error("EP #%s failed, skipping: %s", ep_num, __import__("traceback").format_exc()) + continue if results: logger.info("\n" + "=" * 60) @@ -250,9 +255,14 @@ async def cmd_verify(args: argparse.Namespace) -> None: results: list[BenchmarkResult] = [] for i, case in enumerate(config.cases, 1): logger.info("\n>>> EP %d of %d <<<", i, len(config.cases)) - result = await run_single_ep(case, config, output_dir, "verify", force=args.force) - if result: - results.append(result) + try: + result = await run_single_ep(case, config, output_dir, "verify", force=args.force) + if result: + results.append(result) + except Exception: + ep_num = _ep_num(case.ep_url) + logger.error("EP #%s failed, skipping: %s", ep_num, __import__("traceback").format_exc()) + continue if results: logger.info("\n" + "=" * 60) diff --git a/benchmark/config.yaml b/benchmark/config.yaml index 866d208..df120dc 100644 --- a/benchmark/config.yaml +++ b/benchmark/config.yaml @@ -1,15 +1,52 @@ benchmark_cases: + - ep_url: "https://github.com/openshift/enhancements/pull/1863" + repo_url: "https://github.com/openshift/zero-trust-workload-identity-manager" + description: "SPIRE federation support" + implementation_prs: [68, 82] + - ep_url: "https://github.com/openshift/enhancements/pull/1834" repo_url: "https://github.com/openshift/external-secrets-operator" - description: "External Secrets Operator enhancement" + description: "Network Policy for ESO" implementation_prs: [67, 74] + - ep_url: "https://github.com/openshift/enhancements/pull/1898" + repo_url: "https://github.com/openshift/external-secrets-operator" + description: "ESO install-time customizations" + implementation_prs: [91, 94, 97, 106, 111] + + - ep_url: "https://github.com/openshift/enhancements/pull/1914" + repo_url: "https://github.com/openshift/cert-manager-operator" + description: "TrustManager support in cert-manager" + implementation_prs: [362, 371, 379] + + - ep_url: "https://github.com/openshift/enhancements/pull/1923" + repo_url: "https://github.com/openshift/must-gather-operator" + description: "since/sinceTime fields in MustGather CR" + implementation_prs: [323] + + - ep_url: "https://github.com/openshift/enhancements/pull/1903" + repo_url: "https://github.com/openshift/must-gather-operator" + description: "Remove proxy config from MustGather CR" + implementation_prs: [313] + + - ep_url: "https://github.com/openshift/enhancements/pull/1824" + repo_url: "https://github.com/openshift/zero-trust-workload-identity-manager" + description: "OIDC discovery provider route" + implementation_prs: [33] + + - ep_url: "https://github.com/openshift/enhancements/pull/1906" + repo_url: "https://github.com/openshift/must-gather-operator" + description: "Custom image option in MustGather spec" + implementation_prs: [322] + + - ep_url: "https://github.com/openshift/enhancements/pull/1839" + repo_url: "https://github.com/openshift/must-gather-operator" + description: "PV to persist must-gather data" + implementation_prs: [294] + +# EP #1964 excluded: openshift/must-gather is Shell-based, not a Go operator + settings: - tools_to_benchmark: - - api-generate - - api-implement - iterations: 3 - output_dir: "benchmark/results" - parallel: false model: "claude-opus-4-6" effort: "max" + output_dir: "benchmark/results" diff --git a/benchmark/isolate.py b/benchmark/isolate.py index 32df3bd..3309e8c 100644 --- a/benchmark/isolate.py +++ b/benchmark/isolate.py @@ -15,9 +15,20 @@ logger = logging.getLogger(__name__) -def _run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: +def _run(cmd: list[str], retries: int = 3, **kwargs) -> subprocess.CompletedProcess: logger.debug("Running: %s", " ".join(cmd)) - return subprocess.run(cmd, capture_output=True, text=True, check=True, **kwargs) + for attempt in range(retries): + result = subprocess.run(cmd, capture_output=True, text=True, **kwargs) + if result.returncode == 0: + return result + if attempt < retries - 1: + import time + wait = 5 * (attempt + 1) + logger.warning("Command failed (attempt %d/%d), retrying in %ds: %s", + attempt + 1, retries, wait, result.stderr[:200]) + time.sleep(wait) + result.check_returncode() + return result def _gh_json(args: list[str]) -> dict | list: diff --git a/benchmark/runner.py b/benchmark/runner.py index 918312f..db5b484 100644 --- a/benchmark/runner.py +++ b/benchmark/runner.py @@ -1,10 +1,8 @@ -"""Benchmark runner with iterative feedback loop. +"""Benchmark runner for OAPE tools. -Each iteration: generate code -> compare with ground truth -> improve the OAPE -tool (commands/skills) -> re-generate with improved tool. The tool itself gets -better after each iteration. - -All agents use claude-opus-4-6 in max effort mode. +Provides: +- run_single_iteration: Run OAPE tools once in an isolated environment +- improve_tool_from_all_results: Analyze cross-EP patterns and make concise improvements """ import json @@ -15,8 +13,7 @@ import traceback from pathlib import Path -from compare import compare_iteration -from models import GenerationResult, GroundTruth, IsolatedEnv, IterationScore, ToolImprovement +from models import GenerationResult, IsolatedEnv, ToolImprovement logger = logging.getLogger(__name__) @@ -110,137 +107,6 @@ def _diff_tool_files(plugin_dir: str, backup_dir: Path, before_label: str) -> di return diffs -def _build_improvement_prompt( - iteration: int, - score: IterationScore, - gen_result: GenerationResult, - truth: GroundTruth, - plugin_dir: str, -) -> str: - """Build the prompt for the improver agent.""" - tool_contents: dict[str, str] = {} - for rel in TOOL_FILES: - path = Path(plugin_dir) / rel - if path.exists(): - tool_contents[rel] = path.read_text() - - truth_files_summary = [] - for f in truth.files_added[:20]: - truth_files_summary.append(f" ADDED: {f}") - for f in truth.files_modified[:20]: - truth_files_summary.append(f" MODIFIED: {f}") - - gen_files_summary = [] - for f in gen_result.files_created[:20]: - gen_files_summary.append(f" CREATED: {f}") - for f in gen_result.files_modified[:20]: - gen_files_summary.append(f" MODIFIED: {f}") - - truth_diff_preview = truth.combined_diff[:12000] - gen_diff_preview = gen_result.diff[:12000] - - fc = score.file_classification - classification_summary = [] - if fc.auto_generated: - classification_summary.append(f" Auto-generated artifacts (NOT errors): {fc.auto_generated}") - if fc.formatting_only: - classification_summary.append(f" Formatting-only changes (NOT errors): {fc.formatting_only}") - if fc.valuable_extra: - classification_summary.append(f" Valuable extras (tool outperformed human): {fc.valuable_extra}") - if fc.genuinely_wrong: - classification_summary.append(f" GENUINELY WRONG (these are the real errors): {fc.genuinely_wrong}") - classification_text = chr(10).join(classification_summary) if classification_summary else " No extra files generated." - - tool_sections = [] - for rel, content in tool_contents.items(): - tool_sections.append(f"### FILE: {rel}\n```markdown\n{content}\n```") - - return f"""You are an expert at improving AI code generation tools for OpenShift operators. - -You are given a comparison between what an OAPE tool generated vs what a human actually -shipped for a specific Enhancement Proposal. Your job is to improve the tool's instruction -files so it performs better on ALL future EPs -- not just this specific one. - -## CRITICAL: Make GENERIC improvements only - -- DO NOT add EP-specific guidance (e.g., "when generating federation support, do X"). -- DO NOT reference specific struct names, field names, or feature names from this EP. -- DO add GENERAL PATTERNS that would have caught the issues seen here. -- DO improve the tool's ability to discover what files to generate from ANY EP. -- Think: "What general rule or pattern was missing that caused this gap?" - -## Iteration {iteration} Results - -### Scores -- Completeness: {score.completeness:.1f}% (what % of ground truth structs/fields/functions were generated) -- Convention Compliance: {score.convention_compliance:.1f}% -- Build Success: {"PASS" if score.build_success else "FAIL"} -- Files matched: {score.files_matched} | Files missed: {score.files_missed} -- Genuinely wrong files: {score.genuinely_wrong} | Valuable extras: {score.valuable_extras} | Auto-generated: {score.auto_generated} - -### Understanding Extra Files - -Not all "extra" files are errors: - -{classification_text} - -Files like `zz_generated.deepcopy.go`, CRD YAMLs, `bundle.Dockerfile` are auto-generated -by `make` commands -- generating them is CORRECT. Test files, sample configs, and dedicated -handler files may be the tool doing BETTER than the human. - -Only "GENUINELY WRONG" files are actual errors. - -### Ground Truth (what the human produced) -{chr(10).join(truth_files_summary)} - -Ground truth diff (first 12000 chars): -```diff -{truth_diff_preview} -``` - -### Generated Output (what the tool produced) -{chr(10).join(gen_files_summary)} - -Generated diff (first 12000 chars): -```diff -{gen_diff_preview} -``` - -## Current OAPE Tool Instructions - -{chr(10).join(tool_sections)} - -## Your Task - -Identify GENERAL PATTERNS the tool is missing, then make targeted edits: - -### Priority 1: General patterns for missing files -If the tool missed files (e.g., routes, validation, services, tests), add GENERIC -rules about when an EP implies these file types are needed. Example of a good generic -improvement: "When the EP describes exposing an endpoint, generate a routes.go file -following the existing naming pattern in the controller package." - -### Priority 2: General patterns for struct/field generation -If structs or fields don't match, improve the GENERAL conventions about how to derive -struct shapes from EP descriptions. - -### Priority 3: General patterns for markers and validation -If markers were missing, add GENERAL rules about which markers to always include. - -### Priority 4: General controller patterns -If controller logic was incomplete, add GENERAL reconciliation patterns. - -### What NOT to do: -- Do NOT add feature-specific rules (no "for federation..." or "for rotation...") -- Do NOT reference specific struct/field names from this EP -- Do NOT reduce file coverage -- generating extra tests/samples is GOOD -- Do NOT tell the tool to skip `make generate`/`make manifests` -- Do NOT rewrite entire files -- make surgical, targeted edits - -After editing, explain what GENERAL pattern you added and why. -""" - - async def run_single_iteration( ep_url: str, source_env: IsolatedEnv, @@ -382,96 +248,6 @@ async def run_single_iteration( ) -async def improve_tool( - iteration: int, - gen_result: GenerationResult, - truth: GroundTruth, - score: IterationScore, - plugin_dir: str = PLUGIN_DIR, - backup_dir: Path | None = None, - model: str = "claude-opus-4-6", - effort: str = "max", -) -> ToolImprovement: - """Analyze weaknesses and edit OAPE tool files to improve next iteration.""" - logger.info("=== Improving tool after iteration %d (model=%s, effort=%s) ===", - iteration, model, effort) - - if backup_dir: - _backup_tool_files(plugin_dir, backup_dir, f"before-improvement-{iteration}") - - prompt = _build_improvement_prompt(iteration, score, gen_result, truth, plugin_dir) - improvement_log: list[str] = [] - cost_usd = 0.0 - - try: - from claude_agent_sdk import ( - query, - ClaudeAgentOptions, - AssistantMessage, - ResultMessage, - TextBlock, - ToolUseBlock, - ) - - options = ClaudeAgentOptions( - model=model, - effort=effort, - system_prompt=( - "You are an expert at improving AI code generation tools. " - "You will analyze comparison results and make targeted edits to " - "OAPE tool instruction files (markdown) to improve code generation quality. " - "Focus on specific, actionable improvements. Do NOT rewrite entire files. " - "Make surgical edits that address observed weaknesses." - ), - cwd=plugin_dir, - permission_mode="bypassPermissions", - allowed_tools=[ - "Read", "Write", "Edit", "MultiEdit", - "Bash", "Glob", "Grep", - ], - ) - - async for message in query(prompt=prompt, options=options): - if isinstance(message, AssistantMessage): - for block in message.content: - if isinstance(block, TextBlock): - improvement_log.append(block.text) - elif isinstance(block, ToolUseBlock): - improvement_log.append(f"[tool:{block.name}]") - elif isinstance(message, ResultMessage): - cost_usd = message.total_cost_usd - if message.result: - improvement_log.append(message.result) - - except ImportError: - logger.warning("claude_agent_sdk not available for improvement step") - improvement_log.append("[DRY RUN] claude_agent_sdk not installed") - except Exception: - improvement_log.append(f"[ERROR] {traceback.format_exc()}") - logger.error("Tool improvement failed: %s", traceback.format_exc()) - - diffs = {} - if backup_dir: - diffs = _diff_tool_files(plugin_dir, backup_dir, f"before-improvement-{iteration}") - _backup_tool_files(plugin_dir, backup_dir, f"after-improvement-{iteration}") - - if diffs: - logger.info("Tool improved: %d files changed", len(diffs)) - for fname, diff in diffs.items(): - added = sum(1 for l in diff.split("\n") if l.startswith("+") and not l.startswith("+++")) - removed = sum(1 for l in diff.split("\n") if l.startswith("-") and not l.startswith("---")) - logger.info(" %s: +%d/-%d lines", fname, added, removed) - else: - logger.info("No tool file changes detected after improvement step") - - return ToolImprovement( - iteration=iteration, - files_changed=diffs, - analysis_summary="\n".join(improvement_log[-5:]) if improvement_log else "", - improvement_cost_usd=cost_usd, - ) - - async def improve_tool_from_all_results( results_data: list[dict], truth_data: dict[str, dict], @@ -515,7 +291,7 @@ async def improve_tool_from_all_results( - Build: {"PASS" if s.get('build_success') else "FAIL"} - Matched: {s.get('files_matched', 0)} | Missed: {s.get('files_missed', 0)} | Wrong: {s.get('genuinely_wrong', 0)} | Extras: {s.get('valuable_extras', 0)} - Genuinely wrong files: {fc.get('genuinely_wrong', [])} -- Missed ground truth files (first 15): {(rd.get('implementation_prs', []))} +- Genuinely wrong files: {fc.get('genuinely_wrong', [])} Ground truth diff (first 5000 chars): ```diff @@ -623,73 +399,3 @@ async def improve_tool_from_all_results( ) -async def run_feedback_loop( - ep_url: str, - source_env: IsolatedEnv, - truth: GroundTruth, - num_iterations: int = 3, - plugin_dir: str = PLUGIN_DIR, - backup_dir: Path | None = None, - model: str = "claude-opus-4-6", - effort: str = "max", -) -> tuple[list[GenerationResult], list[ToolImprovement]]: - """Run iterative feedback loop: generate -> compare -> improve -> repeat. - - Each iteration uses the improved tool from the previous iteration. - Returns generation results and tool improvements. - """ - results: list[GenerationResult] = [] - improvements: list[ToolImprovement] = [] - - if backup_dir: - _backup_tool_files(plugin_dir, backup_dir, "original") - - for i in range(1, num_iterations + 1): - logger.info("=" * 50) - logger.info("FEEDBACK LOOP: Iteration %d of %d", i, num_iterations) - logger.info("=" * 50) - - result = await run_single_iteration( - ep_url, source_env, i, - plugin_dir=plugin_dir, model=model, effort=effort, - ) - results.append(result) - - score, outperf = compare_iteration(result, truth) - logger.info( - "Iteration %d scores: completeness=%.1f%% precision=%.1f%% build=%s", - i, score.completeness, score.precision, score.build_success, - ) - - if i < num_iterations: - logger.info("Analyzing weaknesses and improving tool...") - improvement = await improve_tool( - iteration=i, - gen_result=result, - truth=truth, - score=score, - plugin_dir=plugin_dir, - backup_dir=backup_dir, - model=model, - effort=effort, - ) - improvements.append(improvement) - logger.info( - "Tool improvement complete (cost=$%.4f). %d files changed.", - improvement.improvement_cost_usd, len(improvement.files_changed), - ) - else: - logger.info("Final iteration complete. No further improvements.") - - if backup_dir: - original_dir = backup_dir / "original" - if original_dir.exists(): - logger.info("Restoring original tool files...") - for rel in TOOL_FILES: - src = original_dir / rel - dst = Path(plugin_dir) / rel - if src.exists(): - shutil.copy2(src, dst) - logger.info("Original tool files restored.") - - return results, improvements From 03e1cb3bc90a868d53b49c65e0e9d5fbaab59da7 Mon Sep 17 00:00:00 2001 From: Raushan Singh Date: Mon, 11 May 2026 14:45:22 +0530 Subject: [PATCH 6/6] fix: completeness now accounts for missed files Previously, completeness was calculated only across matched files (files both the tool and human touched), ignoring missed files entirely. This gave inflated scores -- e.g., 97.2% when only 4 out of 31 Go files were matched. Now, every missed Go file in the ground truth contributes a 0% score to the average. If the tool matches 4 files at 97% but misses 10 files, completeness = (4*97% + 10*0%) / 14 = 27.7%, not 97%. This gives an honest picture of how much of the total implementation the tool actually covered. Co-authored-by: Cursor --- benchmark/compare.py | 7 ++++--- benchmark/report.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmark/compare.py b/benchmark/compare.py index c00a9e2..790c26e 100644 --- a/benchmark/compare.py +++ b/benchmark/compare.py @@ -403,12 +403,13 @@ def compare_iteration( ), )) + missed_go_files = go_truth_files - go_gen_files + for _missed in missed_go_files: + completeness_scores.append(0.0) + completeness = statistics.mean(completeness_scores) * 100 if completeness_scores else 0.0 convention = statistics.mean(convention_scores) * 100 if convention_scores else 0.0 - if not matched_files and go_truth_files: - completeness = 0.0 - score = IterationScore( iteration=gen_result.iteration, completeness=round(completeness, 1), diff --git a/benchmark/report.py b/benchmark/report.py index c3789d2..31f2162 100644 --- a/benchmark/report.py +++ b/benchmark/report.py @@ -63,7 +63,7 @@ def generate_ep_report( lines.append("") lines.append("**Metric definitions:**") lines.append("") - lines.append("- **Completeness**: What % of the human's structs, fields, and functions did the tool also generate? Higher = tool covered more of the ground truth.") + lines.append("- **Completeness**: What % of the human's Go implementation did the tool cover? Accounts for ALL Go files in the ground truth -- matched files are scored by struct/field/function overlap, missed files count as 0%.") lines.append("- **Convention**: What % of kubebuilder markers (`+kubebuilder:validation:Required`, `+optional`, etc.) match between generated and ground truth?") lines.append("- **Build**: Did `make build` pass on the generated code?") lines.append("- **Matched**: Files the tool generated that also exist in the human implementation (true positives).")