From 309538ff1dff20aef00e0f7f077f8bd774410b48 Mon Sep 17 00:00:00 2001 From: Ariel Rodriguez Date: Sat, 7 Feb 2026 11:13:30 +0100 Subject: [PATCH 01/17] chore: test --- skills/ps-error-handling-design/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/ps-error-handling-design/SKILL.md b/skills/ps-error-handling-design/SKILL.md index 504a85e..9478417 100644 --- a/skills/ps-error-handling-design/SKILL.md +++ b/skills/ps-error-handling-design/SKILL.md @@ -4,7 +4,7 @@ description: Design systems with explicit error handling. Avoid throwing excepti severity: WARN --- -# Principle +## Principle Treat error handling as a first-class design concern, not an afterthought: From b3f440d7717923e7ca55bfc583097ba7706abfb6 Mon Sep 17 00:00:00 2001 From: Ariel Rodriguez Date: Sat, 7 Feb 2026 15:13:57 +0100 Subject: [PATCH 02/17] chore: Finalise benchmark page and workflow. Add codex. --- .github/workflows/benchmark-dashboard.yml | 66 +--- .github/workflows/skill-validation.yml | 5 - ci/generate_basic_html.py | 282 ++++-------------- ci/generate_dashboard_data.py | 132 ++++++-- ci/matrix_generator.py | 16 +- ci/orphan_branch_manager.py | 24 +- ci/publish_benchmarks.py | 90 +++++- ci/scripts/app.js | 254 ++++++++++++++++ docs/benchmarks.json | 4 - docs/index.html | 341 --------------------- docs/specs/benchmark-page.md | 347 +++++++++++++--------- docs/specs/benchmark-workflow.md | 21 +- tests/adapters/__init__.py | 2 + tests/adapters/codex.py | 84 ++++++ tests/adapters/ollama.py | 12 +- tests/domain/types.py | 1 + tests/evaluator.py | 30 +- 17 files changed, 876 insertions(+), 835 deletions(-) create mode 100644 ci/scripts/app.js delete mode 100644 docs/benchmarks.json delete mode 100644 docs/index.html create mode 100644 tests/adapters/codex.py diff --git a/.github/workflows/benchmark-dashboard.yml b/.github/workflows/benchmark-dashboard.yml index 20d2e44..a318d4a 100644 --- a/.github/workflows/benchmark-dashboard.yml +++ b/.github/workflows/benchmark-dashboard.yml @@ -21,88 +21,40 @@ on: type: string jobs: - run-benchmark: + run-benchmark-and-publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: ref: main - path: workspace + path: repo - uses: astral-sh/setup-uv@v5 with: enable-cache: true - name: Run benchmark - working-directory: workspace + working-directory: repo env: OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} run: | - # Install dependencies uv sync --project tests - # Run benchmark uv run --project tests tests/evaluator.py \ --provider ${{ inputs.provider }} \ --model ${{ inputs.model }} \ --judge \ --verbose \ --report - # Rename artifact for clarity - if [ -d tests/results ]; then - ARTIFACT_NAME="benchmark-${{ inputs.provider }}-${{ inputs.model }}-$(date +%Y%m%d-%H%M%S)" - mv tests/results "tests/${ARTIFACT_NAME}" - fi - # Create docs/benchmarks if it doesn't exist for publish_benchmarks.py - mkdir -p docs/benchmarks - - name: Generate dashboard - working-directory: workspace + - name: Publish to benchmark-history branch + working-directory: repo run: | + # Run publish_benchmarks.py with correct paths uv run --project tests python3 ci/publish_benchmarks.py \ --provider ${{ inputs.provider }} \ --model ${{ inputs.model }} \ --branch benchmark-history \ - --no-benchmark - - - name: Generate dashboard - working-directory: workspace - run: | - uv run --project tests python3 ci/publish_benchmarks.py \ - --provider ${{ inputs.provider }} \ - --model ${{ inputs.model }} \ - --branch benchmark-history - - deploy-pages: - needs: run-benchmark - runs-on: ubuntu-latest - - steps: - - name: Checkout workspace - uses: actions/checkout@v4 - with: - ref: main - path: workspace - - - name: Checkout benchmark data - uses: actions/checkout@v4 - with: - ref: benchmark-history - path: benchmark-data - - - name: Copy results to docs - run: | - mkdir -p workspace/docs/benchmarks - cp benchmark-data/docs/benchmarks.json workspace/docs/benchmarks.json 2>/dev/null || true - cp benchmark-data/docs/index.html workspace/docs/index.html 2>/dev/null || true - # Also copy individual benchmark results if they exist - cp -r benchmark-data/docs/benchmarks/*.json workspace/docs/benchmarks/ 2>/dev/null || true - - - name: Commit and push updates - working-directory: workspace - run: | - git config user.name "GitHub Actions" - git config user.email "actions@github.com" - git add docs/ - git commit -m "Update benchmark data" || echo "No changes to commit" - git push origin HEAD:benchmark-history + --no-benchmark \ + --results-dir "tests/results" \ + --output-dir "docs/benchmarks" diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml index 2151882..0dbddb7 100644 --- a/.github/workflows/skill-validation.yml +++ b/.github/workflows/skill-validation.yml @@ -107,11 +107,6 @@ jobs: # Construct arguments array for safety ARGS=(--provider "${{ matrix.provider }}" --model "${{ matrix.model }}" --judge --verbose --report --threshold 50) - if [ -n "${{ matrix.extra_args }}" ]; then - # Split extra_args safely if needed, but for now assuming simple flags - ARGS+=(${{ matrix.extra_args }}) - fi - if [ -n "${{ matrix.skill }}" ]; then ARGS+=(--skill "${{ matrix.skill }}") else diff --git a/ci/generate_basic_html.py b/ci/generate_basic_html.py index e94fc14..48fa7e9 100644 --- a/ci/generate_basic_html.py +++ b/ci/generate_basic_html.py @@ -5,76 +5,20 @@ This is a pure function - no side effects. """ -import json from pathlib import Path -from typing import Any -def generate_html_page(data: dict) -> str: +def generate_html_page(data_src: str, script_src: str) -> str: """ Generate HTML page from benchmark data. Args: - data: Aggregated benchmark data + data_src: Relative path to benchmark JSON + script_src: Relative path to app.js Returns: HTML string """ - summary = data.get('summary', {}) - benchmarks = data.get('benchmarks', []) - unique_skills = data.get('unique_skills', []) - provider_models = data.get('provider_models', []) - - # Build provider/model options - provider_options = "" - seen_providers = set() - for prov, model in provider_models: - if prov not in seen_providers: - provider_options += f'' - seen_providers.add(prov) - - # Build skill rows - skill_rows = "" - for benchmark in benchmarks: - for skill in benchmark.get('skills', []): - skill_name = skill.get('skill_name', 'unknown') - provider = skill.get('provider', 'unknown') - model = skill.get('model', 'unknown') - baseline_rating = skill.get('baseline_rating', 'vague') - skill_rating = skill.get('skill_rating', 'vague') - improvement = skill.get('improvement', 'neutral') - before_code = skill.get('before_code', '// No code generated') - after_code = skill.get('after_code', '// No code generated') - reasoning = skill.get('reasoning', 'No reasoning') - - # Get rating colors - baseline_color = get_rating_color(baseline_rating) - skill_color = get_rating_color(skill_rating) - - # Escape code for HTML - before_code_escaped = escape_html(before_code) - after_code_escaped = escape_html(after_code) - - # Format reasoning for HTML - reasoning_formatted = format_reasoning(reasoning) - - skill_rows += f''' - - {skill_name} - {provider.capitalize()} - {model} - {baseline_rating} - {skill_rating} - {improvement.capitalize()} - - - ''' - - # Build unique skills filter options - skills_filter = '' - for skill_name in unique_skills: - skills_filter += f'' - html = f''' @@ -82,6 +26,7 @@ def generate_html_page(data: dict) -> str: Programming Skills Benchmarks + - +
-
+
Total Benchmarks
-

{summary.get('total_benchmarks', 0)}

+

0

@@ -154,7 +103,7 @@ def generate_html_page(data: dict) -> str:
Skills Tested
-

{summary.get('total_skills', 0)}

+

0

@@ -162,7 +111,7 @@ def generate_html_page(data: dict) -> str:
Improvements
-

{summary.get('improvements', 0)}

+

0

@@ -170,7 +119,7 @@ def generate_html_page(data: dict) -> str:
Improvement Rate
-

{summary.get('improvement_rate', 0)}%

+

0%

@@ -184,20 +133,18 @@ def generate_html_page(data: dict) -> str:
@@ -233,7 +180,6 @@ def generate_html_page(data: dict) -> str: - {skill_rows}
@@ -250,174 +196,60 @@ def generate_html_page(data: dict) -> str:
-
- +
+
Judge Assessment
+
+
+

Before (Baseline):

+

After (With Skill):

+
+
+

Overall Better:

+

Judge Score: /100

+
+
+
+
Reasoning:
-
-
- -
-
- - - - - - - - + + + + ''' return html -def generate_basic_html(input_file: Path, output_file: Path) -> bool: +def generate_basic_html(input_file: Path, output_file: Path, data_src: str = "benchmarks.json", script_src: str = "scripts/app.js") -> bool: """ Main function: Generate HTML page from benchmark JSON. @@ -431,12 +263,8 @@ def generate_basic_html(input_file: Path, output_file: Path) -> bool: True if successful, False otherwise """ try: - # Read input - with open(input_file, 'r', encoding='utf-8') as f: - data = json.load(f) - # Generate HTML - html = generate_html_page(data) + html = generate_html_page(data_src, script_src) # Write output output_file.parent.mkdir(parents=True, exist_ok=True) @@ -457,8 +285,8 @@ def generate_basic_html(input_file: Path, output_file: Path) -> bool: import sys # Default paths - input_file = Path("docs/benchmarks.json") - output_file = Path("docs/index.html") + input_file = Path("docs/benchmarks/benchmarks.json") + output_file = Path("docs/benchmarks/index.html") # Allow command line overrides if len(sys.argv) > 1: diff --git a/ci/generate_dashboard_data.py b/ci/generate_dashboard_data.py index efe1567..e97c9b2 100644 --- a/ci/generate_dashboard_data.py +++ b/ci/generate_dashboard_data.py @@ -8,7 +8,6 @@ import json import re from pathlib import Path -from typing import Any def extract_judgment_reasoning(judgment: dict) -> str: @@ -99,6 +98,50 @@ def normalize_rating(rating: str) -> str: return "vague" +def _parse_benchmark_id(benchmark_id: str) -> tuple[str | None, str | None, str | None]: + """ + Parse benchmark_id in the format: provider-model-YYYYMMDD-HHMMSS + + Returns: + (provider, model, timestamp) or (None, None, None) if not matched. + """ + match = re.match(r'^(?P[^-]+)-(?P.+)-(?P\d{8}-\d{6})$', benchmark_id) + if not match: + return None, None, None + return match.group('provider'), match.group('model'), match.group('timestamp') + + +def _normalize_timestamp(timestamp: str) -> str: + """ + Normalize timestamp to ISO format (YYYY-MM-DDTHH:MM:SS). + """ + if not timestamp: + return '' + + iso_match = re.match(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(Z)?$', timestamp) + if iso_match: + return f"{iso_match.group(1)}-{iso_match.group(2)}-{iso_match.group(3)}T{iso_match.group(4)}:{iso_match.group(5)}:{iso_match.group(6)}" + + compact_match = re.match(r'^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$', timestamp) + if compact_match: + return f"{compact_match.group(1)}-{compact_match.group(2)}-{compact_match.group(3)}T{compact_match.group(4)}:{compact_match.group(5)}:{compact_match.group(6)}" + + dashed_match = re.match(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})$', timestamp) + if dashed_match: + return f"{dashed_match.group(1)}-{dashed_match.group(2)}-{dashed_match.group(3)}T{dashed_match.group(4)}:{dashed_match.group(5)}:{dashed_match.group(6)}" + + return timestamp + + +def _benchmark_id_from_path(file_path: Path) -> str: + """ + Determine benchmark_id based on file path. + """ + if file_path.name == "summary.json": + return file_path.parent.name + return file_path.stem + + def process_benchmark_file(file_path: Path) -> dict | None: """ Process a single benchmark JSON file and extract structured data. @@ -116,32 +159,48 @@ def process_benchmark_file(file_path: Path) -> dict | None: print(f"Error reading {file_path}: {e}") return None - # Extract timestamp from filename or data - timestamp = data.get('timestamp', '') + benchmark_id = _benchmark_id_from_path(file_path) + + # Extract timestamp from summary or benchmark id + timestamp = _normalize_timestamp(data.get('timestamp', '')) + if not timestamp: + _, _, parsed_timestamp = _parse_benchmark_id(benchmark_id) + if parsed_timestamp: + timestamp = _normalize_timestamp(parsed_timestamp) if not timestamp: - # Extract from filename if no timestamp in data match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})', file_path.stem) if match: - timestamp = match.group(1).replace('T', ' ') - - # Get provider and model from filename or data - filename = file_path.stem - if 'ollama' in filename: - provider = 'ollama' - elif 'copilot' in filename: - provider = 'copilot' - elif 'gemini' in filename: - provider = 'gemini' - else: - provider = 'unknown' - - # Extract model name from filename - # Pattern: benchmark-ollama-modelname-timestamp - model_match = re.search(r'benchmark-[^-]+-([^--]+)-\d{4}', filename) - if model_match: - model = model_match.group(1) - else: - model = 'unknown' + timestamp = _normalize_timestamp(match.group(1)) + + # Get provider and model from JSON data (primary source) + results = data.get('results', []) + + provider = 'unknown' + model = 'unknown' + + if results: + model = results[0].get('model', 'unknown') + + parsed_provider, parsed_model, _ = _parse_benchmark_id(benchmark_id) + if provider == 'unknown' and parsed_provider: + provider = parsed_provider + if (not model or model == 'unknown') and parsed_model: + model = parsed_model + + if (not model or model == 'unknown'): + model_match = re.search(r'benchmark-[^-]+-(.+)-\d{4}', benchmark_id) + if model_match: + model = model_match.group(1) + + if provider == 'unknown': + if 'cloud' in str(model).lower(): + provider = 'ollama' + elif 'ollama' in benchmark_id: + provider = 'ollama' + elif 'copilot' in benchmark_id: + provider = 'copilot' + elif 'gemini' in benchmark_id: + provider = 'gemini' # Get results from data results = data.get('results', []) @@ -182,7 +241,7 @@ def process_benchmark_file(file_path: Path) -> dict | None: skills.append(skill_data) return { - 'benchmark_id': file_path.stem, + 'benchmark_id': benchmark_id, 'timestamp': timestamp, 'provider': provider, 'model': model, @@ -223,6 +282,21 @@ def collect_all_benchmarks(benchmarks_dir: Path) -> list[dict]: return all_data +def _write_per_run_data(benchmarks_dir: Path) -> None: + """ + Write per-run data.json next to each summary.json file. + """ + for summary_file in benchmarks_dir.glob('**/summary.json'): + benchmark = process_benchmark_file(summary_file) + if not benchmark: + continue + per_run = build_aggregated_data([benchmark]) + output_file = summary_file.parent / "data.json" + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(per_run, f, indent=2) + + def build_aggregated_data(benchmarks: list[dict]) -> dict: """ Build aggregated data from individual benchmarks. @@ -283,7 +357,8 @@ def generate_dashboard_data(benchmarks_dir: Path, output_file: Path) -> bool: """ Main function: Generate dashboard data from benchmark files. - Pure function - only reads from benchmarks_dir, writes to output_file. + Pure function - only reads from benchmarks_dir, writes to output_file + and per-run data.json files next to summary.json. Args: benchmarks_dir: Directory containing benchmark JSON files @@ -299,6 +374,9 @@ def generate_dashboard_data(benchmarks_dir: Path, output_file: Path) -> bool: # Build aggregated data aggregated = build_aggregated_data(benchmarks) + # Write per-run data.json files + _write_per_run_data(benchmarks_dir) + # Write output output_file.parent.mkdir(parents=True, exist_ok=True) with open(output_file, 'w', encoding='utf-8') as f: @@ -320,7 +398,7 @@ def generate_dashboard_data(benchmarks_dir: Path, output_file: Path) -> bool: # Default paths benchmarks_dir = Path("docs/benchmarks") - output_file = Path("docs/benchmarks.json") + output_file = Path("docs/benchmarks/benchmarks.json") # Allow command line overrides if len(sys.argv) > 1: diff --git a/ci/matrix_generator.py b/ci/matrix_generator.py index 1ea8537..6f4fffa 100644 --- a/ci/matrix_generator.py +++ b/ci/matrix_generator.py @@ -73,15 +73,15 @@ def generate_matrix(config: Dict[str, Any], filter_provider: str = "all") -> Dic "display_name": f"{provider_name}/{model}" } - # Add provider-specific arguments - extra_args = [] - if provider_name == "ollama": - if not provider_config.get("local", True): - extra_args.append("--ollama-cloud") - - item["extra_args"] = " ".join(extra_args) + # Add provider-specific arguments (now handled implicitly in adapter) + # extra_args = [] + # if provider_name == "ollama": + # if not provider_config.get("local", True): + # extra_args.append("--ollama-cloud") + + # item["extra_args"] = " ".join(extra_args) matrix["include"].append(item) - + return matrix diff --git a/ci/orphan_branch_manager.py b/ci/orphan_branch_manager.py index 7c79fac..9955d2e 100644 --- a/ci/orphan_branch_manager.py +++ b/ci/orphan_branch_manager.py @@ -239,12 +239,24 @@ def manage_benchmark_branch( if not result.success: print(f"Warning: Could not clean untracked files: {result.message}") - # Copy docs to branch - result = manager.copy_files_to_branch(docs_dir) - if not result.success: - print(f"Error copying files: {result.message}") - return False - print(f"Copied docs to branch") + # Copy docs to branch (if docs exists) + if docs_dir.exists(): + result = manager.copy_files_to_branch(docs_dir) + if not result.success: + print(f"Error copying files: {result.message}") + return False + print(f"Copied docs to branch") + else: + print(f"Warning: docs directory does not exist: {docs_dir}") + print("Creating minimal docs structure...") + # Create a minimal docs folder with a README + docs_dir.mkdir(parents=True, exist_ok=True) + (docs_dir / "README.md").write_text("# Benchmark Data\n\nThis branch contains benchmark results for the programming skills project.\n") + result = manager.copy_files_to_branch(docs_dir) + if not result.success: + print(f"Error copying files: {result.message}") + return False + print(f"Copied docs to branch") # Add and commit result = manager.add_all_files() diff --git a/ci/publish_benchmarks.py b/ci/publish_benchmarks.py index abd2a88..f751270 100644 --- a/ci/publish_benchmarks.py +++ b/ci/publish_benchmarks.py @@ -7,8 +7,12 @@ """ import argparse +import json +import shutil import subprocess import sys +import time +from datetime import datetime from pathlib import Path @@ -44,7 +48,35 @@ def run_benchmark(provider: str, model: str, skill: str | None = None) -> bool: return result.returncode == 0 -def collect_and_generate(benchmarks_dir: Path, output_dir: Path) -> bool: +def _timestamp_for_id(timestamp: str | None) -> str: + """ + Convert ISO timestamp to YYYYMMDD-HHMMSS format for benchmark_id. + """ + if timestamp: + normalized = timestamp.rstrip("Z") + try: + dt = datetime.fromisoformat(normalized) + return dt.strftime("%Y%m%d-%H%M%S") + except ValueError: + pass + return time.strftime("%Y%m%d-%H%M%S") + + +def _load_summary_timestamp(summary_path: Path) -> str | None: + if not summary_path.exists(): + return None + try: + data = json.loads(summary_path.read_text(encoding="utf-8")) + return data.get("timestamp") + except Exception: + return None + + +def _safe_model_for_id(model: str) -> str: + return model.replace("/", "-") + + +def collect_and_generate(benchmarks_dir: Path, output_dir: Path, script_src: Path) -> bool: """ Collect benchmark data and generate HTML. @@ -58,18 +90,33 @@ def collect_and_generate(benchmarks_dir: Path, output_dir: Path) -> bool: from generate_dashboard_data import generate_dashboard_data from generate_basic_html import generate_basic_html - # Generate aggregated JSON + # Generate aggregated JSON and per-run data.json files data_file = output_dir / "benchmarks.json" if not generate_dashboard_data(benchmarks_dir, data_file): print("Error generating dashboard data") return False - # Generate HTML + # Copy app.js to output scripts directory + scripts_dir = output_dir / "scripts" + scripts_dir.mkdir(parents=True, exist_ok=True) + if script_src.exists(): + shutil.copy2(script_src, scripts_dir / "app.js") + else: + print(f"Warning: app.js not found at {script_src}") + + # Generate aggregate HTML html_file = output_dir / "index.html" - if not generate_basic_html(data_file, html_file): + if not generate_basic_html(data_file, html_file, data_src="benchmarks.json", script_src="scripts/app.js"): print("Error generating HTML") return False + # Generate per-run HTML pages + for run_data_file in output_dir.glob("*/data.json"): + run_html = run_data_file.parent / "index.html" + if not generate_basic_html(run_data_file, run_html, data_src="data.json", script_src="../scripts/app.js"): + print(f"Error generating HTML for {run_data_file.parent}") + return False + return True @@ -99,12 +146,23 @@ def main() -> int: parser.add_argument("--branch", default="benchmark-history", help="Orphan branch name") parser.add_argument("--no-benchmark", action="store_true", help="Skip benchmark run") parser.add_argument("--no-push", action="store_true", help="Skip pushing to orphan branch") + parser.add_argument("--benchmarks-dir", help="Directory containing benchmark JSON files (defaults to output dir)") + parser.add_argument("--results-dir", help="Directory containing summary.json (default: tests/results)") + parser.add_argument("--output-dir", help="Directory to write generated files") args = parser.parse_args() repo_path = Path(__file__).parent.parent - benchmarks_dir = repo_path / "tests" / "results" - docs_dir = repo_path / "docs" + results_dir = Path(args.results_dir) if args.results_dir else repo_path / "tests" / "results" + docs_dir = Path(args.output_dir) if args.output_dir else repo_path / "docs" / "benchmarks" + benchmarks_dir = Path(args.benchmarks_dir) if args.benchmarks_dir else docs_dir + + if benchmarks_dir != docs_dir: + print("Error: --benchmarks-dir must match --output-dir for this workflow") + return 1 + + # Ensure docs directory exists (may not exist if no docs yet) + docs_dir.mkdir(parents=True, exist_ok=True) # Step 1: Run benchmark if requested if not args.no_benchmark: @@ -114,13 +172,27 @@ def main() -> int: else: print("Skipping benchmark run") - # Step 2: Generate dashboard data and HTML + # Step 2: Copy summary.json into versioned run folder + summary_path = results_dir / "summary.json" + if not summary_path.exists(): + print(f"Summary not found at {summary_path}") + return 1 + + timestamp = _load_summary_timestamp(summary_path) + timestamp_id = _timestamp_for_id(timestamp) + benchmark_id = f"{args.provider}-{_safe_model_for_id(args.model)}-{timestamp_id}" + run_dir = docs_dir / benchmark_id + run_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(summary_path, run_dir / "summary.json") + + # Step 3: Generate dashboard data and HTML print("\nGenerating dashboard data...") - if not collect_and_generate(benchmarks_dir, docs_dir): + script_src = Path(__file__).parent / "scripts" / "app.js" + if not collect_and_generate(benchmarks_dir, docs_dir, script_src): print("Dashboard generation failed") return 1 - # Step 3: Push to orphan branch if requested + # Step 4: Push to orphan branch if requested if not args.no_push: print("\nPushing to orphan branch...") if not push_to_orphan_branch(repo_path, docs_dir, args.branch): diff --git a/ci/scripts/app.js b/ci/scripts/app.js new file mode 100644 index 0000000..43af2da --- /dev/null +++ b/ci/scripts/app.js @@ -0,0 +1,254 @@ +let allData = null; +let flatSkills = []; + +function getDataSrc() { + return document.body.dataset.benchmarksSrc || 'benchmarks.json'; +} + +function getRatingColor(rating) { + const colors = { + vague: 'secondary', + regular: 'warning', + good: 'primary', + outstanding: 'success' + }; + return colors[rating] || 'secondary'; +} + +function getImprovementColor(improvement) { + const colors = { + yes: 'success', + no: 'danger', + neutral: 'secondary' + }; + return colors[improvement] || 'secondary'; +} + +function escapeHtml(text) { + if (!text) return ''; + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function clearSelect(selectEl, defaultLabel) { + while (selectEl.options.length > 0) { + selectEl.remove(0); + } + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = defaultLabel; + selectEl.appendChild(opt); +} + +async function fetchData() { + const src = getDataSrc(); + const response = await fetch(src, { cache: 'no-store' }); + if (!response.ok) { + throw new Error(`Failed to load ${src}: ${response.status}`); + } + return response.json(); +} + +function renderSummary(summary) { + const totalBenchmarks = document.getElementById('summary-total-benchmarks'); + const totalSkills = document.getElementById('summary-total-skills'); + const improvements = document.getElementById('summary-improvements'); + const improvementRate = document.getElementById('summary-improvement-rate'); + + if (totalBenchmarks) totalBenchmarks.textContent = summary.total_benchmarks ?? 0; + if (totalSkills) totalSkills.textContent = summary.total_skills ?? 0; + if (improvements) improvements.textContent = summary.improvements ?? 0; + if (improvementRate) improvementRate.textContent = `${summary.improvement_rate ?? 0}%`; +} + +function populateFilters(data) { + const providerSelect = document.getElementById('filter-provider'); + const modelSelect = document.getElementById('filter-model'); + const skillSelect = document.getElementById('filter-skill'); + + const providers = new Set(); + const models = new Set(); + const skills = new Set(); + + (data.benchmarks || []).forEach((benchmark) => { + if (benchmark.provider) providers.add(benchmark.provider); + if (benchmark.model) models.add(benchmark.model); + (benchmark.skills || []).forEach((skill) => { + if (skill.skill_name) skills.add(skill.skill_name); + }); + }); + + if (providerSelect) { + clearSelect(providerSelect, 'All Providers'); + Array.from(providers).sort().forEach((provider) => { + const opt = document.createElement('option'); + opt.value = provider; + opt.textContent = provider.charAt(0).toUpperCase() + provider.slice(1); + providerSelect.appendChild(opt); + }); + } + + if (modelSelect) { + clearSelect(modelSelect, 'All Models'); + Array.from(models).sort().forEach((model) => { + const opt = document.createElement('option'); + opt.value = model; + opt.textContent = model; + modelSelect.appendChild(opt); + }); + } + + if (skillSelect) { + clearSelect(skillSelect, 'All Skills'); + Array.from(skills).sort().forEach((skill) => { + const opt = document.createElement('option'); + opt.value = skill; + opt.textContent = skill; + skillSelect.appendChild(opt); + }); + } +} + +function renderTable(benchmarks) { + flatSkills = []; + const tbody = document.getElementById('skills-table-body'); + if (!tbody) return; + tbody.innerHTML = ''; + + benchmarks.forEach((benchmark) => { + (benchmark.skills || []).forEach((skill) => { + const normalizedSkill = { + ...skill, + provider: skill.provider || benchmark.provider || 'unknown', + model: skill.model || benchmark.model || 'unknown' + }; + const index = flatSkills.length; + flatSkills.push(normalizedSkill); + + const row = document.createElement('tr'); + row.dataset.provider = (normalizedSkill.provider || '').toLowerCase(); + row.dataset.model = (normalizedSkill.model || '').toLowerCase(); + row.dataset.skill = (normalizedSkill.skill_name || '').toLowerCase(); + row.dataset.improvement = (normalizedSkill.improvement || '').toLowerCase(); + + const baselineRating = normalizedSkill.baseline_rating || 'vague'; + const skillRating = normalizedSkill.skill_rating || 'vague'; + const improvement = normalizedSkill.improvement || 'neutral'; + + const improvementLabel = improvement === 'yes' + ? 'Improvement' + : improvement === 'no' + ? 'Regression' + : 'Neutral'; + + row.innerHTML = ` + ${escapeHtml(normalizedSkill.skill_name || 'unknown')} + ${escapeHtml(normalizedSkill.provider || 'unknown')} + ${escapeHtml(normalizedSkill.model || 'unknown')} + ${escapeHtml(baselineRating)} + ${escapeHtml(skillRating)} + ${improvementLabel} + + `; + + const button = row.querySelector('button'); + if (button) { + button.addEventListener('click', () => showDetails(index)); + } + + tbody.appendChild(row); + }); + }); +} + +function filterTable() { + const provider = (document.getElementById('filter-provider')?.value || '').toLowerCase(); + const model = (document.getElementById('filter-model')?.value || '').toLowerCase(); + const skill = (document.getElementById('filter-skill')?.value || '').toLowerCase(); + const improvement = (document.getElementById('filter-improvement')?.value || '').toLowerCase(); + + const rows = document.querySelectorAll('#skills-table-body tr'); + rows.forEach((row) => { + const matchProvider = !provider || row.dataset.provider === provider; + const matchModel = !model || row.dataset.model === model; + const matchSkill = !skill || row.dataset.skill === skill; + const matchImprovement = !improvement || row.dataset.improvement === improvement; + + row.style.display = matchProvider && matchModel && matchSkill && matchImprovement ? '' : 'none'; + }); +} + +function showDetails(index) { + const skill = flatSkills[index]; + if (!skill) return; + + const modalTitle = document.getElementById('detailModalTitle'); + if (modalTitle) { + modalTitle.textContent = `Skill Details: ${skill.skill_name || 'Unknown Skill'}`; + } + + const judgment = skill.judgment || {}; + const beforeRating = judgment.option_a_rating || skill.baseline_rating || 'vague'; + const afterRating = judgment.option_b_rating || skill.skill_rating || 'vague'; + const overallBetter = judgment.overall_better || 'Equal'; + const score = judgment.score ?? 0; + const reasoning = skill.reasoning || judgment.reasoning || 'No reasoning provided'; + + const beforeRatingEl = document.getElementById('modal-before-rating'); + const afterRatingEl = document.getElementById('modal-after-rating'); + const betterEl = document.getElementById('modal-better'); + const scoreEl = document.getElementById('modal-score'); + const reasoningEl = document.getElementById('modal-reasoning'); + const codeBeforeEl = document.getElementById('code-before'); + const codeAfterEl = document.getElementById('code-after'); + + if (beforeRatingEl) beforeRatingEl.textContent = beforeRating; + if (afterRatingEl) afterRatingEl.textContent = afterRating; + if (betterEl) betterEl.textContent = overallBetter; + if (scoreEl) scoreEl.textContent = score; + if (reasoningEl) reasoningEl.innerHTML = escapeHtml(reasoning).replace(/\n/g, '
'); + if (codeBeforeEl) codeBeforeEl.textContent = skill.before_code || '// No code generated'; + if (codeAfterEl) codeAfterEl.textContent = skill.after_code || '// No code generated'; + + const modalEl = document.getElementById('detailModal'); + if (modalEl) { + const modal = new bootstrap.Modal(modalEl); + modal.show(); + if (window.Prism) { + Prism.highlightAllUnder(modalEl); + } + } +} + +function toggleExpand(id, button) { + const container = document.getElementById(id); + if (!container) return; + const expanded = container.classList.toggle('expanded'); + if (button) { + button.textContent = expanded ? 'Collapse' : 'Expand'; + } +} + +async function refreshDashboard() { + await loadDashboard(); +} + +async function loadDashboard() { + try { + const data = await fetchData(); + allData = data; + renderSummary(data.summary || {}); + populateFilters(data); + renderTable(data.benchmarks || []); + } catch (error) { + console.error('Failed to load dashboard data', error); + } +} + +document.addEventListener('DOMContentLoaded', () => { + loadDashboard(); +}); diff --git a/docs/benchmarks.json b/docs/benchmarks.json deleted file mode 100644 index 89e1d34..0000000 --- a/docs/benchmarks.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "benchmarks": [], - "summary": {} -} \ No newline at end of file diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 701f881..0000000 --- a/docs/index.html +++ /dev/null @@ -1,341 +0,0 @@ - - - - - - Programming Skills Benchmarks - - - - - - - - -
-
-
-
-
-
Total Benchmarks
-

0

-
-
-
-
-
-
-
Skills Tested
-

0

-
-
-
-
-
-
-
Improvements
-

0

-
-
-
-
-
-
-
Improvement Rate
-

0%

-
-
-
-
-
- - -
-
-
- - -
-
- - -
-
- - -
-
- - -
-
-
- - -
-
-
-
Benchmark Results
-
-
-
- - - - - - - - - - - - - - - -
SkillProviderModelBeforeAfterImprovementView Details
-
-
-
-
- - - - - - - - - - - \ No newline at end of file diff --git a/docs/specs/benchmark-page.md b/docs/specs/benchmark-page.md index db0d2ef..1fd0a22 100644 --- a/docs/specs/benchmark-page.md +++ b/docs/specs/benchmark-page.md @@ -9,103 +9,156 @@ This spec describes the HTML/CSS/JS structure for the benchmark dashboard page. - **HTML5**: Semantic structure - **Bootstrap 5**: CSS framework for layout and components - **Vanilla JavaScript**: No build step, pure client-side +- **Prism.js (CDN)**: For syntax highlighting in code blocks - **Chart.js (CDN)**: For future time series charts - **Diff Match Patch (CDN)**: For code comparison highlighting -## Page Structure +## Directory Structure + +``` +docs/ +└── benchmarks/ + ├── {benchmark_id}/ # Each run gets its own folder + │ ├── summary.json # Full benchmark results + │ ├── data.json # Extracted data for dashboard + │ └── index.html # Individual page for this run + ├── index.html # Main dashboard (aggregated) + └── benchmarks.json # Aggregated data for JS +``` + +## Page Structure (index.html) ```html - + + + + Programming Skills Benchmarks + - + + + + + - +
-
+
+ +
+
+ + +
+
-
-
-
Total Benchmarks
-

0

-
-
+ +
-
-
-
Skills Tested
-

0

-
-
+ +
-
-
-
-
Last Updated
-

-

-
-
+
+ + +
+
+ +
- +
-
-
-
- Filter by: - - +
+
+
Benchmark Results
+
+
+
+ + + + + + + + + + + + + + + +
SkillProviderModelBeforeAfterImprovementDetails
- -
- - - - - - - - - - - - - - - -
SkillProviderModelBeforeAfterImprovementView Details
-
-