next version

Author: zocomputer

.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "optimize-anything",
-  "version": "0.3.1",
+  "version": "0.3.5",
   "description": "Optimize any text artifact using gepa — prompts, code, configs, skills",
   "keywords": ["optimization", "gepa", "prompts", "evaluator"],
   "author": {

.gitignore

@@ -27,3 +27,5 @@ dogfood_runs/
 PROMPT.md
 .coverage
 run.log
+ROADMAP*
+runs/

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## v0.3.5 - 2026-03-20
+
+### Structural refactor
+- Extracted `cli.py` monolith (1,639 lines) into 4 focused modules:
+  - `preflight.py` — evaluator preflight validation checks
+  - `persist.py` — run-directory persistence, diff output, artifact writing
+  - `cli_optimize.py` — optimize subcommand implementation
+  - `cli_tools.py` — score, validate, analyze, explain, budget, intake, generate-evaluator subcommands
+- `cli.py` reduced to ~500 lines: argparse dispatcher + shared utilities
+
+### New features
+- Added `budget_utilization` section to optimize output contract with `requested`, `evaluator_calls`, `candidates_accepted`, and `efficiency` fields for transparent budget tracking
+
+### Verification
+- All 307 unit tests pass
+- Doc contract and score check gates pass
+- No circular import issues with extracted modules
+
 ## v0.3.2 - 2026-03-09
 
 ### Improvements

CONCEPTS.md

@@ -98,3 +98,4 @@ These are independent axes:
 - `plateau_detected`, `plateau_guidance`
 - optional `evaluator_failure_signal`
 - optional `early_stopped`, `stopped_at_iteration`
+- `budget_utilization` — breakdown of requested budget vs actual evaluator calls, accepted candidates, and efficiency ratio

README.md

@@ -130,6 +130,25 @@ optimize-anything optimize seed.txt \
 
 optimize-anything is also a Claude Code plugin with guided slash commands and skills.
 
+### Plugin Regression Workflow
+
+Use the regression harness when you want to verify that Claude can actually invoke the plugin correctly end-to-end, not merely that the CLI itself still works.
+
+```bash
+# Direct plugin regression run
+uv run python scripts/plugin_regression.py
+
+# Full repo validation including plugin regression
+uv run python scripts/check.py --with-plugin
+```
+
+Requirements:
+- `claude` CLI installed and authenticated
+- `OPENAI_API_KEY` set in the shell that launches the command
+- `ANTHROPIC_API_KEY` set in the shell that launches the command
+
+The harness runs three real scenarios (`analyze`, `validate`, `quick`), saves Claude JSON outputs plus stderr logs, and fails if Claude does not execute the expected workflow or the optimized artifact is not written.
+
 ### Installation
 
 ```bash
@@ -193,6 +212,7 @@ CLI stdout returns a JSON summary with these fields:
 - `plateau_detected`, `plateau_guidance`
 - optional `evaluator_failure_signal`
 - optional `early_stopped`, `stopped_at_iteration`
+- `budget_utilization` (`requested`, `evaluator_calls`, `candidates_accepted`, `efficiency`)
 
 ### Evaluator Protocol (v2)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "optimize-anything"
-version = "0.3.2"
+version = "0.3.5"
 description = "Optimize any text artifact using gepa — prompts, code, configs, skills"
 readme = "README.md"
 license = {text = "MIT"}

scripts/check.py

@@ -34,14 +34,20 @@ def _run_gate(label: str, cmd: list[str], cwd: Path) -> bool:
 
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
-        description="Run all validation gates: pytest, smoke harness, score check."
+        description="Run validation gates: pytest, smoke harness, optional plugin regression, score check."
     )
     parser.add_argument(
         "--skip-smoke",
         action="store_true",
         default=False,
         help="Skip the smoke harness gate (use for offline testing).",
     )
+    parser.add_argument(
+        "--with-plugin",
+        action="store_true",
+        default=False,
+        help="Include the Claude Code plugin regression gate (requires Claude CLI + API keys).",
+    )
     args = parser.parse_args(argv)
 
     root = Path(__file__).parent.parent.resolve()
@@ -68,7 +74,19 @@ def main(argv: list[str] | None = None) -> int:
         )
         results.append(("smoke harness", passed))
 
-    # Gate 3: score check
+    # Gate 3: plugin regression
+    if args.with_plugin:
+        passed = _run_gate(
+            "plugin regression",
+            ["uv", "run", "python", "scripts/plugin_regression.py"],
+            cwd=root,
+        )
+        results.append(("plugin regression", passed))
+    else:
+        print("\n[SKIP] plugin regression (--with-plugin not set)")
+        results.append(("plugin regression", "SKIP"))
+
+    # Gate 4: score check
     passed = _run_gate(
         "score check",
         ["uv", "run", "python", "scripts/score_check.py"],

scripts/plugin_eval.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Plugin evaluation harness for optimize-anything as a Claude Code plugin.
+#
+# Usage:
+#   ./scripts/plugin_eval.sh [scenario]
+#
+# Scenarios:
+#   budget    - Lightweight budget estimation (no LLM calls)
+#   analyze   - Single LLM call to analyze artifact quality dimensions
+#   validate  - Cross-provider validation (2+ providers)
+#   quick     - Full optimization loop (most expensive)
+#   all       - Run all scenarios sequentially
+#
+# Requirements:
+#   - claude CLI installed and authenticated
+#   - OPENAI_API_KEY and/or ANTHROPIC_API_KEY set
+#   - This script must be run from the repo root
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+PLUGIN_DIR="$REPO_ROOT"
+RESULTS_DIR="$REPO_ROOT/runs/plugin-eval"
+SEED_FILE="$REPO_ROOT/runs/zo-eval/seed.txt"
+
+SCENARIO="${1:-all}"
+
+mkdir -p "$RESULTS_DIR"
+
+# Common claude flags
+CLAUDE_BASE=(
+  claude -p
+  --plugin-dir "$PLUGIN_DIR"
+  --output-format json
+  --allowedTools "Bash Read Write Edit Glob Grep"
+  --max-budget-usd 0.50
+)
+
+timestamp() {
+  date -u +"%Y%m%dT%H%M%SZ"
+}
+
+run_scenario() {
+  local name="$1"
+  local prompt="$2"
+  local outfile="$RESULTS_DIR/${name}-$(timestamp).json"
+
+  echo "=== Scenario: $name ==="
+  echo "Prompt: $prompt"
+  echo "Output: $outfile"
+  echo ""
+
+  local start_time
+  start_time=$(date +%s)
+
+  # Run claude in print mode with the plugin loaded
+  "${CLAUDE_BASE[@]}" "$prompt" > "$outfile" 2>"$RESULTS_DIR/${name}-stderr.log" || true
+
+  local end_time
+  end_time=$(date +%s)
+  local elapsed=$(( end_time - start_time ))
+
+  echo "Completed in ${elapsed}s"
+  echo "---"
+
+  # Quick validation: check if output is valid JSON
+  if jq empty "$outfile" 2>/dev/null; then
+    echo "Output: valid JSON"
+    # Extract cost if available
+    jq -r '.cost_usd // "n/a"' "$outfile" 2>/dev/null || true
+  else
+    echo "Output: not valid JSON (check file manually)"
+  fi
+  echo ""
+}
+
+scenario_budget() {
+  run_scenario "budget" \
+    "Use the optimize-anything plugin to estimate a budget for the seed file at $SEED_FILE. Run: optimize-anything budget $SEED_FILE"
+}
+
+scenario_analyze() {
+  run_scenario "analyze" \
+    "Use the optimize-anything plugin to analyze the artifact at $SEED_FILE for quality dimensions. Run: optimize-anything analyze $SEED_FILE --judge-model openai/gpt-4o-mini --objective 'Score the quality of this system prompt'"
+}
+
+scenario_validate() {
+  run_scenario "validate" \
+    "Use the optimize-anything plugin to validate the artifact at $SEED_FILE across multiple providers. Run: optimize-anything validate $SEED_FILE --providers openai/gpt-4o-mini anthropic/claude-haiku-4-5-20251001 --objective 'Score the quality and clarity of this system prompt'"
+}
+
+scenario_quick() {
+  run_scenario "quick" \
+    "Use the optimize-anything plugin to quickly optimize the seed at $SEED_FILE. Run: optimize-anything optimize $SEED_FILE --judge-model openai/gpt-4o-mini --objective 'Improve clarity and specificity of this system prompt' --budget 5 --model openai/gpt-4o-mini --output $RESULTS_DIR/quick-best.txt"
+}
+
+case "$SCENARIO" in
+  budget)   scenario_budget ;;
+  analyze)  scenario_analyze ;;
+  validate) scenario_validate ;;
+  quick)    scenario_quick ;;
+  all)
+    scenario_budget
+    scenario_analyze
+    scenario_validate
+    scenario_quick
+    echo "=== All scenarios complete ==="
+    echo "Results in: $RESULTS_DIR"
+    ;;
+  *)
+    echo "Unknown scenario: $SCENARIO"
+    echo "Valid: budget, analyze, validate, quick, all"
+    exit 1
+    ;;
+esac