diff --git a/.github/workflows/benchmark-dashboard.yml b/.github/workflows/benchmark-dashboard.yml index 20d2e44..f925bd8 100644 --- a/.github/workflows/benchmark-dashboard.yml +++ b/.github/workflows/benchmark-dashboard.yml @@ -21,88 +21,40 @@ on: type: string jobs: - run-benchmark: + run-benchmark-and-publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: ref: main - path: workspace + path: repo - uses: astral-sh/setup-uv@v5 with: enable-cache: true - name: Run benchmark - working-directory: workspace + working-directory: repo env: OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} run: | - # Install dependencies uv sync --project tests - # Run benchmark uv run --project tests tests/evaluator.py \ --provider ${{ inputs.provider }} \ --model ${{ inputs.model }} \ --judge \ --verbose \ - --report - # Rename artifact for clarity - if [ -d tests/results ]; then - ARTIFACT_NAME="benchmark-${{ inputs.provider }}-${{ inputs.model }}-$(date +%Y%m%d-%H%M%S)" - mv tests/results "tests/${ARTIFACT_NAME}" - fi - # Create docs/benchmarks if it doesn't exist for publish_benchmarks.py - mkdir -p docs/benchmarks + --report \ + --all - - name: Generate dashboard - working-directory: workspace + - name: Publish to benchmark-history branch + working-directory: repo run: | + # Run publish_benchmarks.py with correct paths uv run --project tests python3 ci/publish_benchmarks.py \ --provider ${{ inputs.provider }} \ --model ${{ inputs.model }} \ --branch benchmark-history \ - --no-benchmark - - - name: Generate dashboard - working-directory: workspace - run: | - uv run --project tests python3 ci/publish_benchmarks.py \ - --provider ${{ inputs.provider }} \ - --model ${{ inputs.model }} \ - --branch benchmark-history - - deploy-pages: - needs: run-benchmark - runs-on: ubuntu-latest - - steps: - - name: Checkout workspace - uses: actions/checkout@v4 - with: - ref: main - path: workspace - - - name: Checkout benchmark data - uses: actions/checkout@v4 - with: - ref: benchmark-history - path: benchmark-data - - - name: Copy results to docs - run: | - mkdir -p workspace/docs/benchmarks - cp benchmark-data/docs/benchmarks.json workspace/docs/benchmarks.json 2>/dev/null || true - cp benchmark-data/docs/index.html workspace/docs/index.html 2>/dev/null || true - # Also copy individual benchmark results if they exist - cp -r benchmark-data/docs/benchmarks/*.json workspace/docs/benchmarks/ 2>/dev/null || true - - - name: Commit and push updates - working-directory: workspace - run: | - git config user.name "GitHub Actions" - git config user.email "actions@github.com" - git add docs/ - git commit -m "Update benchmark data" || echo "No changes to commit" - git push origin HEAD:benchmark-history + --no-benchmark \ + --output-dir "site/benchmarks" diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml index 2151882..de52512 100644 --- a/.github/workflows/skill-validation.yml +++ b/.github/workflows/skill-validation.yml @@ -107,11 +107,6 @@ jobs: # Construct arguments array for safety ARGS=(--provider "${{ matrix.provider }}" --model "${{ matrix.model }}" --judge --verbose --report --threshold 50) - if [ -n "${{ matrix.extra_args }}" ]; then - # Split extra_args safely if needed, but for now assuming simple flags - ARGS+=(${{ matrix.extra_args }}) - fi - if [ -n "${{ matrix.skill }}" ]; then ARGS+=(--skill "${{ matrix.skill }}") else @@ -128,7 +123,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ steps.artifact.outputs.name }} - path: pr-code/tests/results/ + path: pr-code/tests/data-history/ retention-days: 1 consolidate: @@ -145,10 +140,10 @@ jobs: - uses: actions/download-artifact@v4 with: - path: pr-code/tests/results/ + path: pr-code/tests/data-history/ - name: Consolidate results - run: python3 trusted-scripts/ci/consolidate_results.py --results-dir pr-code/tests/results --output-file pr-code/comment.md + run: python3 trusted-scripts/ci/consolidate_results.py --results-dir pr-code/tests/data-history --output-file pr-code/comment.md - name: Post to PR uses: marocchino/sticky-pull-request-comment@v2 diff --git a/.gitignore b/.gitignore index 9a7f57a..15d9c19 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,6 @@ scratch-* # CI/Validation Artifacts comment.md -results/ + +# Benchmark site output +site/ diff --git a/README.md b/README.md index c117fcd..63be0a4 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ Language-agnostic AI agent skills that enforce fundamental programming principles. This repository provides specific, granular instructions that enable AI coding assistants to produce significantly higher-quality code that adheres to robust engineering standards. +| Dashboard Explorer | Code Comparison | Judge Reasoning | +| :---: | :---: | :---: | +| ![Dashboard](docs/img/dashboard.png) | ![Code Comparison](docs/img/compare-code.png) | ![Judge Results](docs/img/compare-judge-results.png) | + Adopting these skills measurably changes the output of AI models, shifting them from generating merely functional code to producing architecturally sound solutions. ## Table of Contents @@ -15,66 +19,50 @@ Adopting these skills measurably changes the output of AI models, shifting them ## Installation -Select your platform for specific setup instructions: +See: -- [Cursor](docs/install/cursor.md) -- [Antigravity](docs/install/antigravity.md) -- [GitHub Copilot](docs/install/copilot.md) -- [Claude](docs/install/claude.md) +- [Install Instructions](docs/install-instructions.md) ## How it Works The core of this repository is the `skills/` directory. Each skill is encapsulated in its own subdirectory following the `ps-` convention (e.g., `ps-composition-over-coordination`). We use this granular structure because: + 1. **Focus**: It allows the AI to load only the relevant context for a specific task, avoiding context window pollution. 2. **Modularity**: Skills can be improved, versioned, and tested independently. 3. **Composability**: Users can select the specific combination of principles they want to enforce for their project. +## Skill Integration + +Skills should live under the `skills/` directory as `SKILL.md` files. For a full integration guide and documentation index: + +``` +https://agentskills.io/integrate-skills +https://agentskills.io/llms.txt +``` + ## Validation & Testing Every skill is validated against a rigorous testing suite found in the `tests/` directory. - **Automated Judging**: We use an LLM-as-a-Judge approach. The system compares the output of a "Baseline" model (without the skill) against a "Skill" model (with the skill loaded). -- **Semantics over Syntax**: The test does not just look for passing unit tests; it analyzes the *logic* and *structure* of the code. +- **Semantics over Syntax**: The test does not just look for passing unit tests; it analyzes the _logic_ and _structure_ of the code. - **Evidence-Based**: The judge identifies the specific lines of code that demonstrate adherence to or violation of the principle. [Read our Case Study on Judge Fairness](docs/judge-fairness-case-study.md) to see how the system fairly evaluates architectural quality, even when it means failing the Skill model. ## Evaluation Results -Processed 24 evaluation(s). - -| Test Name | Model | Baseline | With Skill | Cases Pass | Winner | -|-----------|-------|----------|------------|------------|--------| -| [results-ollama-devstral-small-2--24b-cloud-ps-composition-over-coordination](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329534502) | devstral-small-2:24b-cloud | good | good | ✅ 2/2 | N/A | -| [results-ollama-devstral-small-2--24b-cloud-ps-error-handling-design](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329534340) | devstral-small-2:24b-cloud | regular | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-explicit-boundaries-adapters](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329535854) | devstral-small-2:24b-cloud | good | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-explicit-ownership-lifecycle](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329535894) | devstral-small-2:24b-cloud | good | good | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-explicit-state-invariants](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329537422) | devstral-small-2:24b-cloud | good | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-functional-core-imperative-shell](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329537046) | devstral-small-2:24b-cloud | regular | good | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-illegal-states-unrepresentable](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329538523) | devstral-small-2:24b-cloud | good | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-local-reasoning](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329538780) | devstral-small-2:24b-cloud | good | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-minimize-mutation](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329540068) | devstral-small-2:24b-cloud | good | good | ✅ 2/2 | N/A | -| [results-ollama-devstral-small-2--24b-cloud-ps-naming-as-design](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329540040) | devstral-small-2:24b-cloud | regular | good | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-policy-mechanism-separation](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329541792) | devstral-small-2:24b-cloud | good | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-devstral-small-2--24b-cloud-ps-single-direction-data-flow](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329541535) | devstral-small-2:24b-cloud | regular | good | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-composition-over-coordination](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329524580) | rnj-1:8b-cloud | outstanding | good | ❌ 2/2 | Baseline | -| [results-ollama-rnj-1--8b-cloud-ps-error-handling-design](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329524126) | rnj-1:8b-cloud | vague | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-explicit-boundaries-adapters](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329526125) | rnj-1:8b-cloud | regular | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-explicit-ownership-lifecycle](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329526263) | rnj-1:8b-cloud | good | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-explicit-state-invariants](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329528479) | rnj-1:8b-cloud | regular | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-functional-core-imperative-shell](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329527817) | rnj-1:8b-cloud | regular | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-illegal-states-unrepresentable](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329529527) | rnj-1:8b-cloud | outstanding | outstanding | ✅ 2/2 | N/A | -| [results-ollama-rnj-1--8b-cloud-ps-local-reasoning](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329529241) | rnj-1:8b-cloud | vague | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-minimize-mutation](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329531124) | rnj-1:8b-cloud | regular | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-naming-as-design](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329531393) | rnj-1:8b-cloud | vague | good | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-policy-mechanism-separation](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329532599) | rnj-1:8b-cloud | regular | outstanding | ✅ 2/2 | With Skill | -| [results-ollama-rnj-1--8b-cloud-ps-single-direction-data-flow](https://github.com/Ariel-Rodriguez/programming-skills/actions/runs/21547621647/artifacts/5329532551) | rnj-1:8b-cloud | vague | good | ✅ 2/2 | With Skill | +Dashboard: + +``` +https://ariel-rodriguez.github.io/programming-skills/ +``` ## Documentation -- [Architecture](docs/architecture.md) - Repository design & structure +- [Architecture](docs/specs/architecture.md) - Repository design & structure - [Contributing](docs/contributing.md) - How to add/modify skills & benchmarks - [AI Prompt Wrapper](docs/ai-prompt-wrapper.md) - Configure your AI assistant - [Changelog](CHANGELOG.md) - Version history & skill changes diff --git a/ci/consolidate_results.py b/ci/consolidate_results.py index 41529f5..aa432fe 100644 --- a/ci/consolidate_results.py +++ b/ci/consolidate_results.py @@ -113,7 +113,7 @@ def main(): parser = argparse.ArgumentParser(description="Consolidate evaluation results") parser.add_argument("--mode", choices=["pr-comment", "benchmark"], default="pr-comment", help="Output mode: pr-comment or benchmark") - parser.add_argument("--results-dir", type=Path, default="tests/results", + parser.add_argument("--results-dir", type=Path, default="tests/data-history", help="Directory containing evaluation results") parser.add_argument("--output-dir", type=Path, default=None, help="Output directory for benchmark mode") @@ -126,8 +126,8 @@ def main(): print(f"==> Consolidating results (mode: {args.mode})") - # Find all summary.json files - summary_files = sorted(args.results_dir.glob("*/summary.json")) + # Find all summary files + summary_files = sorted(args.results_dir.glob("**/summary-*.json")) if not summary_files: print(f"No results found in {args.results_dir}") diff --git a/ci/generate_basic_html.py b/ci/generate_basic_html.py deleted file mode 100644 index e94fc14..0000000 --- a/ci/generate_basic_html.py +++ /dev/null @@ -1,470 +0,0 @@ -""" -Generate Basic HTML - Pure Function - -Generates a Bootstrap-styled HTML page from benchmark data. -This is a pure function - no side effects. -""" - -import json -from pathlib import Path -from typing import Any - - -def generate_html_page(data: dict) -> str: - """ - Generate HTML page from benchmark data. - - Args: - data: Aggregated benchmark data - - Returns: - HTML string - """ - summary = data.get('summary', {}) - benchmarks = data.get('benchmarks', []) - unique_skills = data.get('unique_skills', []) - provider_models = data.get('provider_models', []) - - # Build provider/model options - provider_options = "" - seen_providers = set() - for prov, model in provider_models: - if prov not in seen_providers: - provider_options += f'' - seen_providers.add(prov) - - # Build skill rows - skill_rows = "" - for benchmark in benchmarks: - for skill in benchmark.get('skills', []): - skill_name = skill.get('skill_name', 'unknown') - provider = skill.get('provider', 'unknown') - model = skill.get('model', 'unknown') - baseline_rating = skill.get('baseline_rating', 'vague') - skill_rating = skill.get('skill_rating', 'vague') - improvement = skill.get('improvement', 'neutral') - before_code = skill.get('before_code', '// No code generated') - after_code = skill.get('after_code', '// No code generated') - reasoning = skill.get('reasoning', 'No reasoning') - - # Get rating colors - baseline_color = get_rating_color(baseline_rating) - skill_color = get_rating_color(skill_rating) - - # Escape code for HTML - before_code_escaped = escape_html(before_code) - after_code_escaped = escape_html(after_code) - - # Format reasoning for HTML - reasoning_formatted = format_reasoning(reasoning) - - skill_rows += f''' - - {skill_name} - {provider.capitalize()} - {model} - {baseline_rating} - {skill_rating} - {improvement.capitalize()} - - - ''' - - # Build unique skills filter options - skills_filter = '' - for skill_name in unique_skills: - skills_filter += f'' - - html = f''' - - - - - Programming Skills Benchmarks - - - - - - - - -
-
-
-
-
-
Total Benchmarks
-

{summary.get('total_benchmarks', 0)}

-
-
-
-
-
-
-
Skills Tested
-

{summary.get('total_skills', 0)}

-
-
-
-
-
-
-
Improvements
-

{summary.get('improvements', 0)}

-
-
-
-
-
-
-
Improvement Rate
-

{summary.get('improvement_rate', 0)}%

-
-
-
-
-
- - -
-
-
- - -
-
- - -
-
- - -
-
- - -
-
-
- - -
-
-
-
Benchmark Results
-
-
-
- - - - - - - - - - - - - - {skill_rows} - -
SkillProviderModelBeforeAfterImprovementView Details
-
-
-
-
- - - - - - - - - - -''' - - return html - - -def generate_basic_html(input_file: Path, output_file: Path) -> bool: - """ - Main function: Generate HTML page from benchmark JSON. - - Pure function - only reads from input_file, writes to output_file. - - Args: - input_file: Path to aggregated benchmark JSON - output_file: Path to write HTML file - - Returns: - True if successful, False otherwise - """ - try: - # Read input - with open(input_file, 'r', encoding='utf-8') as f: - data = json.load(f) - - # Generate HTML - html = generate_html_page(data) - - # Write output - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, 'w', encoding='utf-8') as f: - f.write(html) - - print(f"Generated {output_file}") - return True - - except Exception as e: - print(f"Error generating HTML: {e}") - import traceback - traceback.print_exc() - return False - - -if __name__ == "__main__": - import sys - - # Default paths - input_file = Path("docs/benchmarks.json") - output_file = Path("docs/index.html") - - # Allow command line overrides - if len(sys.argv) > 1: - input_file = Path(sys.argv[1]) - if len(sys.argv) > 2: - output_file = Path(sys.argv[2]) - - success = generate_basic_html(input_file, output_file) - sys.exit(0 if success else 1) diff --git a/ci/generate_dashboard_data.py b/ci/generate_dashboard_data.py index efe1567..44b716d 100644 --- a/ci/generate_dashboard_data.py +++ b/ci/generate_dashboard_data.py @@ -8,7 +8,6 @@ import json import re from pathlib import Path -from typing import Any def extract_judgment_reasoning(judgment: dict) -> str: @@ -99,128 +98,156 @@ def normalize_rating(rating: str) -> str: return "vague" -def process_benchmark_file(file_path: Path) -> dict | None: +def _normalize_timestamp(timestamp: str) -> str: """ - Process a single benchmark JSON file and extract structured data. + Normalize timestamp to ISO format (YYYY-MM-DDTHH:MM:SS). + """ + if not timestamp: + return '' + + iso_match = re.match(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(Z)?$', timestamp) + if iso_match: + return f"{iso_match.group(1)}-{iso_match.group(2)}-{iso_match.group(3)}T{iso_match.group(4)}:{iso_match.group(5)}:{iso_match.group(6)}" + + compact_match = re.match(r'^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$', timestamp) + if compact_match: + return f"{compact_match.group(1)}-{compact_match.group(2)}-{compact_match.group(3)}T{compact_match.group(4)}:{compact_match.group(5)}:{compact_match.group(6)}" + + dashed_match = re.match(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})$', timestamp) + if dashed_match: + return f"{dashed_match.group(1)}-{dashed_match.group(2)}-{dashed_match.group(3)}T{dashed_match.group(4)}:{dashed_match.group(5)}:{dashed_match.group(6)}" + + return timestamp + + +def _benchmark_id_from_parts(provider: str, model: str, timestamp: str, fallback_stamp: str | None) -> str: + stamp = _normalize_timestamp(timestamp) + if stamp and "T" in stamp: + stamp = stamp.replace(":", "").replace("-", "").replace("T", "-") + stamp = stamp[:15] + if not stamp and fallback_stamp: + stamp = fallback_stamp + + # Sanitize model name for Windows paths (no colons) + safe_model = str(model).replace(":", "-") + return f"{provider}-{safe_model}-{stamp or 'unknown'}" + + +def collect_all_benchmarks(history_dir: Path) -> list[dict]: + """ + Collect all benchmark data from a directory of per-skill history files. Args: - file_path: Path to the benchmark JSON file + history_dir: Directory containing per-skill history JSON files Returns: - Structured benchmark data dict or None if processing fails + List of structured benchmark data """ - try: - with open(file_path, 'r', encoding='utf-8') as f: - data = json.load(f) - except (json.JSONDecodeError, IOError) as e: - print(f"Error reading {file_path}: {e}") - return None - - # Extract timestamp from filename or data - timestamp = data.get('timestamp', '') - if not timestamp: - # Extract from filename if no timestamp in data - match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})', file_path.stem) + if not history_dir.exists(): + return [] + + benchmarks_map: dict[tuple[str, str, str], dict] = {} + + for skill_file in history_dir.glob('**/*.json'): + if skill_file.name.startswith("summary-"): + continue + try: + data = json.loads(skill_file.read_text()) + except Exception: + continue + + skill_name = data.get('skill', skill_file.parent.name) + model = data.get('model', 'unknown') + provider = data.get('provider', 'unknown') + timestamp = _normalize_timestamp(data.get('timestamp', '')) + + if provider == 'unknown': + if 'sonnet' in str(model).lower(): + provider = 'copilot' + elif 'gpt' in str(model).lower(): + provider = 'codex' + + fallback_stamp = None + match = re.search(r'(\d{8}-\d{6})', skill_file.stem) if match: - timestamp = match.group(1).replace('T', ' ') - - # Get provider and model from filename or data - filename = file_path.stem - if 'ollama' in filename: - provider = 'ollama' - elif 'copilot' in filename: - provider = 'copilot' - elif 'gemini' in filename: - provider = 'gemini' - else: - provider = 'unknown' - - # Extract model name from filename - # Pattern: benchmark-ollama-modelname-timestamp - model_match = re.search(r'benchmark-[^-]+-([^--]+)-\d{4}', filename) - if model_match: - model = model_match.group(1) - else: - model = 'unknown' - - # Get results from data - results = data.get('results', []) - if not results: - return None - - # Build skills list - skills = [] - for result in results: - judgment = result.get('judgment', {}) - - # Get before/after code from test results - test_results = result.get('results', []) + fallback_stamp = match.group(1) + + key = (provider, model, timestamp or fallback_stamp or 'unknown') + benchmark_id = _benchmark_id_from_parts(provider, model, timestamp, fallback_stamp) + + benchmark = benchmarks_map.get(key) + if not benchmark: + benchmark = { + 'benchmark_id': benchmark_id, + 'timestamp': timestamp, + 'provider': provider, + 'model': model, + 'skills': [] + } + benchmarks_map[key] = benchmark + + judgment = data.get('judgment', {}) + test_results = data.get('results', []) before_code, after_code = extract_code_from_test_results(test_results, provider) - # Determine improvement - overall_better = judgment.get('overall_better', 'Equal') - if overall_better == 'B': - improvement = 'yes' - elif overall_better == 'A': - improvement = 'no' + overall_better = judgment.get('overall_better') if judgment else None + if overall_better in ('A', 'B', 'Equal'): + if overall_better == 'B': + improvement = 'yes' + elif overall_better == 'A': + improvement = 'no' + else: + improvement = 'neutral' else: - improvement = 'neutral' + improvement = data.get('improvement') + if isinstance(improvement, (int, float)): + improvement = 'yes' if improvement > 0 else ('no' if improvement < 0 else 'neutral') + elif improvement is None: + improvement = 'neutral' skill_data = { - 'skill_name': result.get('skill', 'unknown'), + 'skill_name': skill_name, + 'skill_version': data.get('skill_version', '1.0.0'), 'provider': provider, 'model': model, 'timestamp': timestamp, - 'baseline_rating': normalize_rating(judgment.get('option_a_rating', '')), - 'skill_rating': normalize_rating(judgment.get('option_b_rating', '')), + 'baseline_rating': normalize_rating(judgment.get('option_a_rating', data.get('baseline_rating', ''))), + 'skill_rating': normalize_rating(judgment.get('option_b_rating', data.get('skill_rating', ''))), 'improvement': improvement, - 'reasoning': extract_judgment_reasoning(judgment), + 'reasoning': extract_judgment_reasoning(judgment) if judgment else 'No reasoning provided', 'before_code': before_code, 'after_code': after_code, - 'judgment': judgment + 'judgment': judgment, + 'judge_error': data.get('judge_error', False), + 'tests': [ + { + 'name': t.get('name', 'unknown'), + 'input': t.get('input', ''), + 'expected': t.get('expected', {}), + 'baseline_response': t.get('baseline', {}).get('response_full', ''), + 'skill_response': t.get('skill', {}).get('response_full', '') + } + for t in test_results + ] } - skills.append(skill_data) + benchmark['skills'].append(skill_data) - return { - 'benchmark_id': file_path.stem, - 'timestamp': timestamp, - 'provider': provider, - 'model': model, - 'skills': skills - } + all_data = list(benchmarks_map.values()) + all_data.sort(key=lambda x: x.get('timestamp', ''), reverse=True) + return all_data -def collect_all_benchmarks(benchmarks_dir: Path) -> list[dict]: +def _write_per_run_data(output_dir: Path, benchmarks: list[dict]) -> None: """ - Collect all benchmark data from a directory. - - Args: - benchmarks_dir: Directory containing benchmark JSON files - - Returns: - List of structured benchmark data + Write per-run data.json for each benchmark. """ - if not benchmarks_dir.exists(): - return [] - - all_data = [] - - # Look for summary.json files - for summary_file in benchmarks_dir.glob('**/summary.json'): - data = process_benchmark_file(summary_file) - if data: - all_data.append(data) - - # Also look for benchmark-*.json files - for benchmark_file in benchmarks_dir.glob('**/benchmark-*.json'): - data = process_benchmark_file(benchmark_file) - if data: - all_data.append(data) - - # Sort by timestamp (newest first) - all_data.sort(key=lambda x: x.get('timestamp', ''), reverse=True) - - return all_data + for benchmark in benchmarks: + benchmark_id = benchmark.get("benchmark_id", "unknown") + per_run = build_aggregated_data([benchmark]) + output_file = output_dir / "data" / benchmark_id / "data.json" + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(per_run, f, indent=2) def build_aggregated_data(benchmarks: list[dict]) -> dict: @@ -245,62 +272,87 @@ def build_aggregated_data(benchmarks: list[dict]) -> dict: skill_names.add(skill.get('skill_name', 'unknown')) provider_models.add((benchmark.get('provider', ''), benchmark.get('model', ''))) - # Count improvements - total_skills = 0 - improvements = 0 - regressions = 0 - neutral = 0 - + # 1. Track latest run per Model + Skill + model_skill_latest: dict[tuple[str, str], dict] = {} + for benchmark in benchmarks: + model = benchmark.get('model', 'unknown') for skill in benchmark.get('skills', []): - total_skills += 1 - if skill.get('improvement') == 'yes': - improvements += 1 - elif skill.get('improvement') == 'no': - regressions += 1 - else: - neutral += 1 - - # Build summary - summary = { - 'total_benchmarks': len(benchmarks), - 'total_skills': total_skills, - 'improvements': improvements, - 'regressions': regressions, - 'neutral': neutral, - 'improvement_rate': round((improvements / total_skills * 100) if total_skills > 0 else 0, 1) - } + skill_name = skill.get('skill_name', 'unknown') + key = (model, skill_name) + + # Since benchmarks are sorted by timestamp desc in collect_all_benchmarks, + # the first one we see for a key is the latest. + if key not in model_skill_latest: + model_skill_latest[key] = skill + + # 2. Group by model to generate leaderboard + model_stats: dict[str, dict] = {} + for (model, skill_name), skill_data in model_skill_latest.items(): + if model not in model_stats: + model_stats[model] = { + 'model': model, + 'provider': skill_data.get('provider', 'unknown'), + 'total_tested': 0, + 'improvements': 0, + 'improvement_rate': 0 + } + + stats = model_stats[model] + stats['total_tested'] += 1 + if skill_data.get('improvement') == 'yes': + stats['improvements'] += 1 + + # 3. Finalize leaderboard + leaderboard = [] + for model_name, stats in model_stats.items(): + if stats['total_tested'] > 0: + stats['improvement_rate'] = round((stats['improvements'] / stats['total_tested'] * 100), 1) + leaderboard.append(stats) + + # Sort leaderboard by rate desc, then total tested desc + leaderboard.sort(key=lambda x: (x['improvement_rate'], x['total_tested']), reverse=True) + + # 4. Global summary (still useful for general totals) + total_benchmarks = len(benchmarks) return { 'benchmarks': benchmarks, - 'summary': summary, + 'summary': { + 'total_benchmarks': total_benchmarks, + 'leaderboard': leaderboard + }, 'unique_skills': sorted(list(skill_names)), 'provider_models': sorted(list(provider_models)) } -def generate_dashboard_data(benchmarks_dir: Path, output_file: Path) -> bool: +def generate_dashboard_data(history_dir: Path, output_dir: Path) -> bool: """ Main function: Generate dashboard data from benchmark files. - Pure function - only reads from benchmarks_dir, writes to output_file. + Pure function - only reads from history_dir, writes to output_dir. Args: - benchmarks_dir: Directory containing benchmark JSON files - output_file: Path to write aggregated data JSON + history_dir: Directory containing per-skill history JSON files + output_dir: Output directory for benchmarks.json and data/* Returns: True if successful, False otherwise """ try: # Collect all benchmarks - benchmarks = collect_all_benchmarks(benchmarks_dir) + benchmarks = collect_all_benchmarks(history_dir) # Build aggregated data aggregated = build_aggregated_data(benchmarks) + # Write per-run data.json files + _write_per_run_data(output_dir, benchmarks) + # Write output - output_file.parent.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "benchmarks.json" with open(output_file, 'w', encoding='utf-8') as f: json.dump(aggregated, f, indent=2) @@ -319,14 +371,14 @@ def generate_dashboard_data(benchmarks_dir: Path, output_file: Path) -> bool: import sys # Default paths - benchmarks_dir = Path("docs/benchmarks") - output_file = Path("docs/benchmarks.json") + results_dir = Path("tests/data-history") + output_dir = Path("site/benchmarks") # Allow command line overrides if len(sys.argv) > 1: - benchmarks_dir = Path(sys.argv[1]) + results_dir = Path(sys.argv[1]) if len(sys.argv) > 2: - output_file = Path(sys.argv[2]) + output_dir = Path(sys.argv[2]) - success = generate_dashboard_data(benchmarks_dir, output_file) + success = generate_dashboard_data(results_dir, output_dir) sys.exit(0 if success else 1) diff --git a/ci/matrix_generator.py b/ci/matrix_generator.py index 1ea8537..6f4fffa 100644 --- a/ci/matrix_generator.py +++ b/ci/matrix_generator.py @@ -73,15 +73,15 @@ def generate_matrix(config: Dict[str, Any], filter_provider: str = "all") -> Dic "display_name": f"{provider_name}/{model}" } - # Add provider-specific arguments - extra_args = [] - if provider_name == "ollama": - if not provider_config.get("local", True): - extra_args.append("--ollama-cloud") - - item["extra_args"] = " ".join(extra_args) + # Add provider-specific arguments (now handled implicitly in adapter) + # extra_args = [] + # if provider_name == "ollama": + # if not provider_config.get("local", True): + # extra_args.append("--ollama-cloud") + + # item["extra_args"] = " ".join(extra_args) matrix["include"].append(item) - + return matrix diff --git a/ci/orchestrate_evaluations.py b/ci/orchestrate_evaluations.py index 7436301..5e9f5ed 100644 --- a/ci/orchestrate_evaluations.py +++ b/ci/orchestrate_evaluations.py @@ -230,7 +230,7 @@ def main(): sys.exit(0) # Clean previous results - results_base = Path("tests/results") + results_base = Path("tests/data-history") import shutil if results_base.exists(): shutil.rmtree(results_base) diff --git a/ci/orphan_branch_manager.py b/ci/orphan_branch_manager.py index 7c79fac..b49b132 100644 --- a/ci/orphan_branch_manager.py +++ b/ci/orphan_branch_manager.py @@ -80,7 +80,7 @@ def branch_exists(self) -> bool: Returns: True if branch exists, False otherwise """ - result = self._run_git("rev-parse", "--verify", self.branch_name, "2>/dev/null") + result = self._run_git("show-ref", "--verify", f"refs/heads/{self.branch_name}") return result.success def create_orphan_branch(self) -> GitResult: @@ -133,7 +133,19 @@ def copy_files_to_branch(self, source_dir: Path, dest_dir: Optional[str] = None) try: import shutil - shutil.copytree(source_dir, dest, dirs_exist_ok=True) + if dest_dir: + shutil.copytree(source_dir, dest, dirs_exist_ok=True) + return GitResult(success=True, message=f"Copied files to {dest}") + + for item in source_dir.iterdir(): + target = dest / item.name + if item.is_dir(): + if target.exists(): + shutil.rmtree(target) + shutil.copytree(item, target) + else: + shutil.copy2(item, target) + return GitResult(success=True, message=f"Copied files to {dest}") except Exception as e: return GitResult(success=False, message=str(e)) @@ -234,17 +246,33 @@ def manage_benchmark_branch( return False print(f"Checked out branch: {branch_name}") - # Clean branch + # Clean branch (remove tracked + untracked) + result = manager._run_git("rm", "-rf", ".") + if not result.success: + print(f"Warning: Could not remove tracked files: {result.message}") + result = manager.clean_untracked() if not result.success: print(f"Warning: Could not clean untracked files: {result.message}") - # Copy docs to branch - result = manager.copy_files_to_branch(docs_dir) - if not result.success: - print(f"Error copying files: {result.message}") - return False - print(f"Copied docs to branch") + # Copy output to branch (if exists) + if docs_dir.exists(): + result = manager.copy_files_to_branch(docs_dir) + if not result.success: + print(f"Error copying files: {result.message}") + return False + print(f"Copied output to branch") + else: + print(f"Warning: output directory does not exist: {docs_dir}") + print("Creating minimal output structure...") + # Create a minimal output folder with a README + docs_dir.mkdir(parents=True, exist_ok=True) + (docs_dir / "README.md").write_text("# Benchmark Data\n\nThis branch contains benchmark results for the programming skills project.\n") + result = manager.copy_files_to_branch(docs_dir) + if not result.success: + print(f"Error copying files: {result.message}") + return False + print(f"Copied output to branch") # Add and commit result = manager.add_all_files() diff --git a/ci/publish_benchmarks.py b/ci/publish_benchmarks.py index abd2a88..8e7deb2 100644 --- a/ci/publish_benchmarks.py +++ b/ci/publish_benchmarks.py @@ -7,9 +7,11 @@ """ import argparse +import shutil import subprocess import sys from pathlib import Path +import tempfile def run_benchmark(provider: str, model: str, skill: str | None = None) -> bool: @@ -44,7 +46,7 @@ def run_benchmark(provider: str, model: str, skill: str | None = None) -> bool: return result.returncode == 0 -def collect_and_generate(benchmarks_dir: Path, output_dir: Path) -> bool: +def collect_and_generate(history_dir: Path, output_dir: Path) -> bool: """ Collect benchmark data and generate HTML. @@ -56,20 +58,11 @@ def collect_and_generate(benchmarks_dir: Path, output_dir: Path) -> bool: True if successful, False otherwise """ from generate_dashboard_data import generate_dashboard_data - from generate_basic_html import generate_basic_html - - # Generate aggregated JSON - data_file = output_dir / "benchmarks.json" - if not generate_dashboard_data(benchmarks_dir, data_file): + # Generate aggregated JSON and per-run data.json files + if not generate_dashboard_data(history_dir, output_dir): print("Error generating dashboard data") return False - # Generate HTML - html_file = output_dir / "index.html" - if not generate_basic_html(data_file, html_file): - print("Error generating HTML") - return False - return True @@ -93,37 +86,69 @@ def push_to_orphan_branch(repo_path: Path, docs_dir: Path, branch_name: str) -> def main() -> int: """Main entry point.""" parser = argparse.ArgumentParser(description="Publish benchmark results") - parser.add_argument("--provider", required=True, help="Provider name") - parser.add_argument("--model", required=True, help="Model name") + parser.add_argument("--provider", help="Provider name") + parser.add_argument("--model", help="Model name") parser.add_argument("--skill", help="Optional specific skill to test") parser.add_argument("--branch", default="benchmark-history", help="Orphan branch name") parser.add_argument("--no-benchmark", action="store_true", help="Skip benchmark run") parser.add_argument("--no-push", action="store_true", help="Skip pushing to orphan branch") + parser.add_argument("--benchmarks-dir", help="Deprecated (unused)") + parser.add_argument("--history-dir", help="Directory containing per-skill history (default: tests/data-history)") + parser.add_argument("--output-dir", help="Directory to write generated files") args = parser.parse_args() repo_path = Path(__file__).parent.parent - benchmarks_dir = repo_path / "tests" / "results" - docs_dir = repo_path / "docs" + if args.benchmarks_dir: + print("Warning: --benchmarks-dir is deprecated and ignored.") + + history_dir = Path(args.history_dir) if args.history_dir else repo_path / "tests" / "data-history" + output_dir = Path(args.output_dir) if args.output_dir else repo_path / "site" / "benchmarks" + + # Ensure docs directory exists (may not exist if no docs yet) + output_dir.mkdir(parents=True, exist_ok=True) # Step 1: Run benchmark if requested if not args.no_benchmark: + if not args.provider or not args.model: + print("Error: --provider and --model are required unless --no-benchmark is set") + return 1 if not run_benchmark(args.provider, args.model, args.skill): print("Benchmark run failed") return 1 else: print("Skipping benchmark run") - # Step 2: Generate dashboard data and HTML + # Step 2: Ensure history directory exists + history_dir.mkdir(parents=True, exist_ok=True) + + # Step 4: Sync static site assets into output directory + source_dir = repo_path / "src" / "pages" / "benchmarks" + if not source_dir.exists(): + print(f"Static site source not found at {source_dir}") + return 1 + + for item in source_dir.iterdir(): + dest = output_dir / item.name + if item.is_dir(): + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(item, dest) + else: + shutil.copy2(item, dest) + + # Step 5: Generate dashboard data files print("\nGenerating dashboard data...") - if not collect_and_generate(benchmarks_dir, docs_dir): + if not collect_and_generate(history_dir, output_dir): print("Dashboard generation failed") return 1 - # Step 3: Push to orphan branch if requested + # Step 6: Push to orphan branch if requested if not args.no_push: print("\nPushing to orphan branch...") - if not push_to_orphan_branch(repo_path, docs_dir, args.branch): + temp_dir = Path(tempfile.mkdtemp(prefix="benchmarks-site-")) + shutil.copytree(output_dir, temp_dir, dirs_exist_ok=True) + if not push_to_orphan_branch(repo_path, temp_dir, args.branch): print("Push to orphan branch failed") return 1 else: diff --git a/ci/run_evaluation.py b/ci/run_evaluation.py index 89705ee..ddbc9d6 100644 --- a/ci/run_evaluation.py +++ b/ci/run_evaluation.py @@ -67,13 +67,13 @@ def main(): timestamp_iso = get_timestamp_iso() model_clean = extract_model_name(model) - # Results directory with timestamp for historical tracking + # History directory with timestamp for historical tracking # Format: provider/model-timestamp - results_dir = Path("tests/results") / provider / f"{model_clean}-{timestamp}" - results_dir.mkdir(parents=True, exist_ok=True) + history_dir = Path("tests/data-history") / provider / f"{model_clean}-{timestamp}" + history_dir.mkdir(parents=True, exist_ok=True) print(f"==> Running evaluation: {provider}/{model}") - print(f" Results: {results_dir}") + print(f" History: {history_dir}") print(f" Timestamp: {timestamp_iso}") print(f" Threshold: {threshold}") @@ -100,8 +100,8 @@ def main(): model, "--threshold", threshold, - "--results-dir", - str(results_dir), + "--history-dir", + str(history_dir), "--report", "--judge", "--verbose", @@ -115,7 +115,7 @@ def main(): cmd.extend(skill_args) print(f"[{provider}/{model}] Executing evaluation...") - print(f"[{provider}/{model}] Output will be in: {results_dir}") + print(f"[{provider}/{model}] Output will be in: {history_dir}") # Execute with live output exit_code = subprocess.run(cmd).returncode @@ -126,8 +126,8 @@ def main(): print(f"[{provider}/{model}] ❌ Evaluation failed with exit code {exit_code}") # Save exit code and metadata for consolidation - (results_dir / "exit_code").write_text(str(exit_code)) - (results_dir / "metadata.json").write_text(json.dumps({ + (history_dir / "exit_code").write_text(str(exit_code)) + (history_dir / "metadata.json").write_text(json.dumps({ "provider": provider, "model": model, "timestamp": timestamp_iso, diff --git a/ci/scripts/app.js b/ci/scripts/app.js new file mode 100644 index 0000000..43af2da --- /dev/null +++ b/ci/scripts/app.js @@ -0,0 +1,254 @@ +let allData = null; +let flatSkills = []; + +function getDataSrc() { + return document.body.dataset.benchmarksSrc || 'benchmarks.json'; +} + +function getRatingColor(rating) { + const colors = { + vague: 'secondary', + regular: 'warning', + good: 'primary', + outstanding: 'success' + }; + return colors[rating] || 'secondary'; +} + +function getImprovementColor(improvement) { + const colors = { + yes: 'success', + no: 'danger', + neutral: 'secondary' + }; + return colors[improvement] || 'secondary'; +} + +function escapeHtml(text) { + if (!text) return ''; + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function clearSelect(selectEl, defaultLabel) { + while (selectEl.options.length > 0) { + selectEl.remove(0); + } + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = defaultLabel; + selectEl.appendChild(opt); +} + +async function fetchData() { + const src = getDataSrc(); + const response = await fetch(src, { cache: 'no-store' }); + if (!response.ok) { + throw new Error(`Failed to load ${src}: ${response.status}`); + } + return response.json(); +} + +function renderSummary(summary) { + const totalBenchmarks = document.getElementById('summary-total-benchmarks'); + const totalSkills = document.getElementById('summary-total-skills'); + const improvements = document.getElementById('summary-improvements'); + const improvementRate = document.getElementById('summary-improvement-rate'); + + if (totalBenchmarks) totalBenchmarks.textContent = summary.total_benchmarks ?? 0; + if (totalSkills) totalSkills.textContent = summary.total_skills ?? 0; + if (improvements) improvements.textContent = summary.improvements ?? 0; + if (improvementRate) improvementRate.textContent = `${summary.improvement_rate ?? 0}%`; +} + +function populateFilters(data) { + const providerSelect = document.getElementById('filter-provider'); + const modelSelect = document.getElementById('filter-model'); + const skillSelect = document.getElementById('filter-skill'); + + const providers = new Set(); + const models = new Set(); + const skills = new Set(); + + (data.benchmarks || []).forEach((benchmark) => { + if (benchmark.provider) providers.add(benchmark.provider); + if (benchmark.model) models.add(benchmark.model); + (benchmark.skills || []).forEach((skill) => { + if (skill.skill_name) skills.add(skill.skill_name); + }); + }); + + if (providerSelect) { + clearSelect(providerSelect, 'All Providers'); + Array.from(providers).sort().forEach((provider) => { + const opt = document.createElement('option'); + opt.value = provider; + opt.textContent = provider.charAt(0).toUpperCase() + provider.slice(1); + providerSelect.appendChild(opt); + }); + } + + if (modelSelect) { + clearSelect(modelSelect, 'All Models'); + Array.from(models).sort().forEach((model) => { + const opt = document.createElement('option'); + opt.value = model; + opt.textContent = model; + modelSelect.appendChild(opt); + }); + } + + if (skillSelect) { + clearSelect(skillSelect, 'All Skills'); + Array.from(skills).sort().forEach((skill) => { + const opt = document.createElement('option'); + opt.value = skill; + opt.textContent = skill; + skillSelect.appendChild(opt); + }); + } +} + +function renderTable(benchmarks) { + flatSkills = []; + const tbody = document.getElementById('skills-table-body'); + if (!tbody) return; + tbody.innerHTML = ''; + + benchmarks.forEach((benchmark) => { + (benchmark.skills || []).forEach((skill) => { + const normalizedSkill = { + ...skill, + provider: skill.provider || benchmark.provider || 'unknown', + model: skill.model || benchmark.model || 'unknown' + }; + const index = flatSkills.length; + flatSkills.push(normalizedSkill); + + const row = document.createElement('tr'); + row.dataset.provider = (normalizedSkill.provider || '').toLowerCase(); + row.dataset.model = (normalizedSkill.model || '').toLowerCase(); + row.dataset.skill = (normalizedSkill.skill_name || '').toLowerCase(); + row.dataset.improvement = (normalizedSkill.improvement || '').toLowerCase(); + + const baselineRating = normalizedSkill.baseline_rating || 'vague'; + const skillRating = normalizedSkill.skill_rating || 'vague'; + const improvement = normalizedSkill.improvement || 'neutral'; + + const improvementLabel = improvement === 'yes' + ? 'Improvement' + : improvement === 'no' + ? 'Regression' + : 'Neutral'; + + row.innerHTML = ` + ${escapeHtml(normalizedSkill.skill_name || 'unknown')} + ${escapeHtml(normalizedSkill.provider || 'unknown')} + ${escapeHtml(normalizedSkill.model || 'unknown')} + ${escapeHtml(baselineRating)} + ${escapeHtml(skillRating)} + ${improvementLabel} + + `; + + const button = row.querySelector('button'); + if (button) { + button.addEventListener('click', () => showDetails(index)); + } + + tbody.appendChild(row); + }); + }); +} + +function filterTable() { + const provider = (document.getElementById('filter-provider')?.value || '').toLowerCase(); + const model = (document.getElementById('filter-model')?.value || '').toLowerCase(); + const skill = (document.getElementById('filter-skill')?.value || '').toLowerCase(); + const improvement = (document.getElementById('filter-improvement')?.value || '').toLowerCase(); + + const rows = document.querySelectorAll('#skills-table-body tr'); + rows.forEach((row) => { + const matchProvider = !provider || row.dataset.provider === provider; + const matchModel = !model || row.dataset.model === model; + const matchSkill = !skill || row.dataset.skill === skill; + const matchImprovement = !improvement || row.dataset.improvement === improvement; + + row.style.display = matchProvider && matchModel && matchSkill && matchImprovement ? '' : 'none'; + }); +} + +function showDetails(index) { + const skill = flatSkills[index]; + if (!skill) return; + + const modalTitle = document.getElementById('detailModalTitle'); + if (modalTitle) { + modalTitle.textContent = `Skill Details: ${skill.skill_name || 'Unknown Skill'}`; + } + + const judgment = skill.judgment || {}; + const beforeRating = judgment.option_a_rating || skill.baseline_rating || 'vague'; + const afterRating = judgment.option_b_rating || skill.skill_rating || 'vague'; + const overallBetter = judgment.overall_better || 'Equal'; + const score = judgment.score ?? 0; + const reasoning = skill.reasoning || judgment.reasoning || 'No reasoning provided'; + + const beforeRatingEl = document.getElementById('modal-before-rating'); + const afterRatingEl = document.getElementById('modal-after-rating'); + const betterEl = document.getElementById('modal-better'); + const scoreEl = document.getElementById('modal-score'); + const reasoningEl = document.getElementById('modal-reasoning'); + const codeBeforeEl = document.getElementById('code-before'); + const codeAfterEl = document.getElementById('code-after'); + + if (beforeRatingEl) beforeRatingEl.textContent = beforeRating; + if (afterRatingEl) afterRatingEl.textContent = afterRating; + if (betterEl) betterEl.textContent = overallBetter; + if (scoreEl) scoreEl.textContent = score; + if (reasoningEl) reasoningEl.innerHTML = escapeHtml(reasoning).replace(/\n/g, '
'); + if (codeBeforeEl) codeBeforeEl.textContent = skill.before_code || '// No code generated'; + if (codeAfterEl) codeAfterEl.textContent = skill.after_code || '// No code generated'; + + const modalEl = document.getElementById('detailModal'); + if (modalEl) { + const modal = new bootstrap.Modal(modalEl); + modal.show(); + if (window.Prism) { + Prism.highlightAllUnder(modalEl); + } + } +} + +function toggleExpand(id, button) { + const container = document.getElementById(id); + if (!container) return; + const expanded = container.classList.toggle('expanded'); + if (button) { + button.textContent = expanded ? 'Collapse' : 'Expand'; + } +} + +async function refreshDashboard() { + await loadDashboard(); +} + +async function loadDashboard() { + try { + const data = await fetchData(); + allData = data; + renderSummary(data.summary || {}); + populateFilters(data); + renderTable(data.benchmarks || []); + } catch (error) { + console.error('Failed to load dashboard data', error); + } +} + +document.addEventListener('DOMContentLoaded', () => { + loadDashboard(); +}); diff --git a/ci/update_skill_versions.py b/ci/update_skill_versions.py new file mode 100644 index 0000000..1ddd757 --- /dev/null +++ b/ci/update_skill_versions.py @@ -0,0 +1,20 @@ +import pathlib +import re + +def update_skills(): + skills_dir = pathlib.Path('skills') + for skill_file in skills_dir.glob('**/SKILL.md'): + print(f"Updating {skill_file}") + content = skill_file.read_text(encoding='utf-8') + + # Add version after severity if it doesn't exist + if 'version:' not in content: + new_content = re.sub( + r'severity:\s*(BLOCK|WARN|SUGGEST)', + r'severity: \1\nversion: 1.0.0', + content + ) + skill_file.write_text(new_content, encoding='utf-8') + +if __name__ == "__main__": + update_skills() diff --git a/docs/ai-prompt-wrapper.md b/docs/ai-prompt-wrapper.md index 99b1b53..4874bed 100644 --- a/docs/ai-prompt-wrapper.md +++ b/docs/ai-prompt-wrapper.md @@ -1,124 +1,36 @@ -# AI Agent System Prompt +# Skills Integration Guide -Use this prompt to configure AI coding assistants to enforce these programming skills. +This repository organizes skills under the `skills/` directory. Each skill lives in its own folder and includes a `SKILL.md` file. -> **Note:** This configuration leverages severity levels (BLOCK/WARN/SUGGEST) for robust enforcement. +## Documentation Index -## For Cursor / Antigravity / Claude +Fetch the complete documentation index at: -Add to your project's `.cursorrules`, `.cursorsettings`, or agent configuration: - -```markdown -# Code Culture Enforcement - -You are an AI coding partner operating under a strict code culture focused on **clarity, correctness, and maintainability** over speed. - -## Core Principles - -Before generating or accepting code, enforce these 12 programming skills: - -### Core Structural Skills (1-3) -1. **Functional Core / Imperative Shell**: Decision logic must be pure. Side effects isolated. -2. **Explicit State & Invariants**: State must be intentionally designed with documented invariants. -3. **Single Direction of Data Flow**: Data ownership must be unambiguous. No circular dependencies. - -### Design Discipline Skills (4-10) -4. **Explicit Boundaries & Adapters**: Isolate frameworks and external systems -5. **Local Reasoning**: Code must be understandable without global search -6. **Naming as Design**: Names encode intent, not implementation -7. **Error Handling as Design**: Errors are modeled explicitly -8. **Policy vs Mechanism**: Separate what from how -9. **Explicit Ownership & Lifecycle**: Clear resource ownership -10. **Minimize Mutation**: Controlled, localized mutation only - -### Refinement Skills (11-12) -11. **Composition Over Coordination**: Prefer composition to orchestration -12. **Illegal States Unrepresentable**: Prevent misuse structurally - -## Generation Rules - -- Always separate decision logic from side effects -- Never place business logic in frameworks, hooks, or effects -- Make state transitions explicit and invariant-preserving -- Prefer clarity and locality over brevity or cleverness -- Reject implicit dependencies and hidden control flow - -## Review Questions - -Before accepting code, ask yourself: -- Who owns this state? -- What invariant is preserved? -- Can this be reasoned about locally? -- Can this logic run without the framework? - -## Uncertainty Handling - -If unsure: -- Default to safer, more explicit design -- Prefer refactoring over patching -- Require explicit developer intent, don't assume - -## Goal - -Eliminate spaghetti code by construction, not by cleanup. ``` - -## For GitHub Copilot - -Add to `.github/copilot/instructions.md`: - -```markdown -When generating code for this project: - -1. **Pure Functions First**: Keep business logic pure and testable -2. **Explicit State**: Document state invariants. Avoid boolean flag explosion -3. **Clear Data Flow**: One source of truth. Unidirectional updates -4. **Framework Boundaries**: Isolate React/Vue/framework code from core logic -5. **Local Understanding**: Avoid hidden dependencies and globals -6. **Intent in Names**: Use domain terms, not generic names -7. **Error as Data**: Model errors explicitly in types/return values -8. **Separate Rules**: Extract business rules from execution mechanisms -9. **Lifecycle Clarity**: Pair creation with cleanup -10. **Minimize Mutation**: Prefer transformations over in-place changes -11. **Compose Simply**: Build complexity from simple units -12. **Design Out Errors**: Make invalid states impossible to represent - -Prioritize correctness and clarity. Ask clarifying questions rather than assuming requirements. +https://agentskills.io/llms.txt ``` -## For General AI Assistants +Use this file to discover all available pages before exploring further. -```markdown -## Code Quality Instructions +## Integrate Skills Into Your Agent -This project values: -- **Functional cores** with imperative shells -- **Explicit state** with preserved invariants -- **Unidirectional data flow** -- **Clear boundaries** between framework and logic -- **Local reasoning** without global context -- **Intent-revealing names** -- **Explicit error handling** -- **Separation of policy and mechanism** -- **Clear ownership and lifecycle** -- **Controlled mutation** -- **Composition over coordination** -- **Structurally safe designs** +Follow the integration guide here: -When reviewing or generating code, enforce these principles consistently. +``` +https://agentskills.io/integrate-skills ``` -## Custom Configuration - -Projects can customize the prompt by: +This guide explains how to: -1. Adding domain-specific rules -2. Providing examples of good/bad patterns -3. Linking to internal style guides -4. Emphasizing specific skills for their context +1. Discover skills in configured directories +2. Load metadata at startup +3. Match tasks to relevant skills +4. Activate skills by loading full instructions +5. Execute scripts and access bundled assets as needed -Place customizations in `.ai-code-culture.md` in your project root. +--- -## Version Information +Notes: -- **Current:** v2.2.0 - Severity-based skills with automated benchmarking. +- Skills are filesystem-based: your agent should read `skills/*/SKILL.md`. +- Keep metadata concise; only load full skill instructions when needed. diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 0e8cc37..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,69 +0,0 @@ -# Architecture - -## Design Principles - -- **Simplicity**: Skills are defined once with clear principles and examples. -- **Single Source of Truth**: The `skills/` directory contains all skill definitions and their associated benchmark scenarios. -- **Language Independence**: Principles are documented in a way that applies to any programming language, typically using pseudocode for illustrations. -- **Severity-Based Enforcement**: Skills are categorized by impact (BLOCK, WARN, SUGGEST) to guide AI agents on how to handle violations. - -## Directory Structure - -```text -basic-programming-skills/ -├── skills/ # Skill definitions -│ ├── ps-functional-core-imperative-shell/ -│ │ ├── SKILL.md # Principle & Documentation -│ │ ├── test.json # Benchmark scenarios & expectations -│ │ └── example_case.js # (Optional) External input files -│ └── ... -├── tests/ # Benchmarking suite -│ ├── evaluator.py # Main evaluation engine -│ └── config.yaml # Benchmarking configuration -├── docs/ # Project documentation -│ ├── architecture.md -│ ├── contributing.md -│ └── ai-prompt-wrapper.md -└── README.md -``` - -## Skill Format (v2.2.0) - -Each skill is a directory containing: - -1. **SKILL.md**: A markdown file with YAML frontmatter containing `name`, `description`, and `severity`. -2. **test.json**: A machine-readable file containing benchmark test cases. -3. **Support Files**: Optional source code files referenced by `test.json` for complex scenarios. - -### Severity Levels - -- **BLOCK**: Architectural or structural violations that must stop code generation and require immediate refactoring. -- **WARN**: Significant design issues that should be flagged and explained, but may be bypassed with justification. -- **SUGGEST**: Optional improvements or refinements that enhance code quality without being critical. - -## Automated Benchmarking - -The project includes a robust benchmarking system to verify that AI models can correctly apply the defined skills. - -### Evaluator Engine - -The `tests/evaluator.py` script: -- Automatically discovers skills and their corresponding `test.json` files. -- Executes tests against models using the Ollama REST API. -- Supports dual-pass evaluation: a baseline run (without skill context) and a skill run (with skill context). -- Calculates the improvement delta and pass rates. - -### Test Schema - -Tests in `test.json` support: -- **`includes`**: Strings that must all be present in the model's response. -- **`excludes`**: Strings that must not appear in the response. -- **`regex`**: Regular expression patterns for advanced verification. -- **`min_length` / `max_length`**: Character count constraints. -- **External Inputs**: Inputs can reference a filename in the same directory, which the evaluator will load dynamically. - -## Platform Support - -The skills are designed to be copied directly into AI agent environments: -- **Cursor**: Copy `skills/` to `.cursor/skills/`. -- **Antigravity**: Copy `skills/` to `.agent/skills/`. diff --git a/docs/benchmarks.json b/docs/benchmarks.json deleted file mode 100644 index 89e1d34..0000000 --- a/docs/benchmarks.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "benchmarks": [], - "summary": {} -} \ No newline at end of file diff --git a/docs/composition-refactor-example.js b/docs/composition-refactor-example.js deleted file mode 100644 index ed825d4..0000000 --- a/docs/composition-refactor-example.js +++ /dev/null @@ -1,214 +0,0 @@ -/** - * REFACTORED: Composition Over Coordination - * - * Instead of one large coordinator, we compose small, focused units. - * Each unit has a single responsibility and can be tested independently. - */ - -// ============================================ -// COMPOSABLE UNITS - Each does ONE thing -// ============================================ - -class AuthenticationResult { - constructor(userId, email, isValid) { - this.userId = userId; - this.email = email; - this.isValid = isValid; - } - - static unauthorized() { - return new AuthenticationResult(null, null, false); - } -} - -function authenticate(authService) { - return async (token) => { - const session = await authService.getSession(token); - if (!session || !session.isValid) { - return AuthenticationResult.unauthorized(); - } - return new AuthenticationResult(session.userId, session.email, true); - }; -} - -function validatePaymentRequest(request) { - if (!request.amount || request.amount <= 0) { - throw new Error("Invalid amount"); - } - if (!request.currency || request.currency !== 'USD') { - throw new Error("Only USD supported"); - } - return request; -} - -function checkRisk(db) { - return async (userId) => { - const riskScore = await db.query( - "SELECT score FROM risk_profiles WHERE user_id = ?", - userId - ); - if (riskScore > 80) { - throw new Error("High risk transaction"); - } - return riskScore; - }; -} - -function chargeBank(bankApi) { - return async (request, userId) => { - const transaction = await bankApi.charge({ - card: request.cardNumber, - amount: request.amount, - ref: `TXN-${Date.now()}` - }); - - if (transaction.status !== 'success') { - throw new Error(`Bank error: ${transaction.errorMessage}`); - } - - return transaction; - }; -} - -function recordTransaction(db) { - return async (transaction, userId, amount) => { - await db.execute( - "UPDATE accounts SET balance = balance - ? WHERE user_id = ?", - [amount, userId] - ); - await db.execute( - "INSERT INTO transactions (id, user_id, amount) VALUES (?, ?, ?)", - [transaction.id, userId, amount] - ); - return transaction; - }; -} - -function sendNotification(emailService) { - return async (email, amount) => { - await emailService.send( - email, - "Payment Successful", - `You charged $${amount}` - ); - }; -} - -// ============================================ -// COMPOSITION - Combine units into a pipeline -// ============================================ - -class PaymentProcessor { - constructor(authService, db, bankApi, emailService) { - // Create composed units - this.authenticate = authenticate(authService); - this.checkRisk = checkRisk(db); - this.chargeBank = chargeBank(bankApi); - this.recordTransaction = recordTransaction(db); - this.sendNotification = sendNotification(emailService); - } - - async processPayment(request) { - // Pipeline: each step transforms or validates data - const auth = await this.authenticate(request.token); - if (!auth.isValid) { - throw new Error("Unauthorized"); - } - - validatePaymentRequest(request); - await this.checkRisk(auth.userId); - - const transaction = await this.chargeBank(request, auth.userId); - await this.recordTransaction(transaction, auth.userId, request.amount); - await this.sendNotification(auth.email, request.amount); - - return { - success: true, - transactionId: transaction.id - }; - } -} - -// ============================================ -// BENEFITS OF THIS APPROACH -// ============================================ - -/** - * ✅ Each unit is small (< 15 lines) and focused - * ✅ Units are testable independently: - * - validatePaymentRequest(request) - pure function - * - authenticate(mockAuth)(token) - easy to mock - * ✅ Reusable: checkRisk() can be used in other flows - * ✅ Clear data flow: input → output for each unit - * ✅ Easy to extend: add fraud detection unit without changing others - * ✅ No deep coupling: units don't reach into each other - */ - -// ============================================ -// EXAMPLE TESTS - Simple because units are small -// ============================================ - -// Test individual units -function testValidation() { - const valid = { amount: 100, currency: 'USD' }; - const result = validatePaymentRequest(valid); - console.assert(result.amount === 100, "Validation should pass"); - - try { - validatePaymentRequest({ amount: 0, currency: 'USD' }); - console.assert(false, "Should throw for zero amount"); - } catch (e) { - console.assert(e.message === "Invalid amount"); - } -} - -// Test composition without real dependencies -async function testPaymentFlow() { - const mockAuth = { - getSession: async (token) => ({ userId: 123, email: 'user@test.com', isValid: true }) - }; - const mockDb = { - query: async () => 50, // Low risk - execute: async () => {} - }; - const mockBank = { - charge: async () => ({ id: 'TXN-001', status: 'success' }) - }; - const mockEmail = { - send: async () => {} - }; - - const processor = new PaymentProcessor(mockAuth, mockDb, mockBank, mockEmail); - const result = await processor.processPayment({ - token: 'abc', - amount: 100, - currency: 'USD', - cardNumber: '4111' - }); - - console.assert(result.success === true); - console.assert(result.transactionId === 'TXN-001'); -} - -// ============================================ -// COMPARISON: Before vs After -// ============================================ - -/** - * BEFORE (Coordination): - * - 1 method, 60+ lines - * - 4 dependencies mixed throughout - * - Hard to test: must mock everything at once - * - Hard to change: touching anything risks breaking everything - * - Hard to reuse: logic trapped in coordinator - * - * AFTER (Composition): - * - 6 focused units, each < 15 lines - * - Dependencies injected per unit - * - Easy to test: mock one unit at a time - * - Easy to change: swap units independently - * - Easy to reuse: units work in other contexts - * - * Main method went from orchestrating details to composing units. - * Behavior emerges from structure, not procedural logic. - */ diff --git a/docs/generate-benchmark.md b/docs/generate-benchmark.md new file mode 100644 index 0000000..e29b251 --- /dev/null +++ b/docs/generate-benchmark.md @@ -0,0 +1,91 @@ +# Generate Benchmark Data + +This guide shows the current end-to-end flow for generating benchmark history and publishing the dashboard. + +## Prerequisites + +- Run from the repo root. +- Codex CLI or Copilot CLI installed (if using those providers). +- `site/` output is generated locally and **not** committed to `main`. + +## 1) Run an Evaluation + +Run a full benchmark run and write results into `tests/data-history/`. + +### Codex + +```bash +uv run --project tests tests/evaluator.py \ + --provider codex \ + --model gpt-5.1-codex-mini \ + --judge --verbose \ + --all +``` + +### Copilot + +```bash +uv run --project tests tests/evaluator.py \ + --provider copilot \ + --model sonnet \ + --judge --verbose \ + --all +``` + +### Ollama + +```bash +uv run --project tests tests/evaluator.py \ + --provider ollama \ + --model llama3.2:latest \ + --judge --verbose \ + --all +``` + +Results are written to: + +``` +tests/data-history/ + summary---.json + /-.json +``` + +## 2) Generate the Site Output + +This copies `src/pages/benchmarks/` into the output root and generates JSON data. + +```bash +python3 ci/publish_benchmarks.py --no-benchmark --no-push --output-dir site/benchmarks +``` + +Output is created at: + +``` +site/benchmarks/ + index.html + app.js + benchmarks.json + data//data.json +``` + +Rerun without `--no-push` to push the orphan branch, and skip following + +## 3) Publish the Orphan Branch + +Either rerun without `--no-push` or push manually: + +```bash +git checkout benchmark-history +git rm -rf . +cp -R site/benchmarks/. . +git add . +git commit -m "Publish benchmark dashboard" +git push origin benchmark-history +git checkout main +``` + +## Notes + +- The dashboard aggregates all runs in `tests/data-history/`. +- The orphan branch is a **flat root** for GitHub Pages. +- If you need to debug outputs, inspect `site/benchmarks/benchmarks.json`. diff --git a/docs/img/compare-code.png b/docs/img/compare-code.png new file mode 100644 index 0000000..0ecb333 Binary files /dev/null and b/docs/img/compare-code.png differ diff --git a/docs/img/compare-judge-results.png b/docs/img/compare-judge-results.png new file mode 100644 index 0000000..df2b6c8 Binary files /dev/null and b/docs/img/compare-judge-results.png differ diff --git a/docs/img/dashboard.png b/docs/img/dashboard.png new file mode 100644 index 0000000..35edea4 Binary files /dev/null and b/docs/img/dashboard.png differ diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 701f881..0000000 --- a/docs/index.html +++ /dev/null @@ -1,341 +0,0 @@ - - - - - - Programming Skills Benchmarks - - - - - - - - -
-
-
-
-
-
Total Benchmarks
-

0

-
-
-
-
-
-
-
Skills Tested
-

0

-
-
-
-
-
-
-
Improvements
-

0

-
-
-
-
-
-
-
Improvement Rate
-

0%

-
-
-
-
-
- - -
-
-
- - -
-
- - -
-
- - -
-
- - -
-
-
- - -
-
-
-
Benchmark Results
-
-
-
- - - - - - - - - - - - - - - -
SkillProviderModelBeforeAfterImprovementView Details
-
-
-
-
- - - - - - - - - - - \ No newline at end of file diff --git a/docs/install-instructions.md b/docs/install-instructions.md new file mode 100644 index 0000000..13ea59e --- /dev/null +++ b/docs/install-instructions.md @@ -0,0 +1,100 @@ +# Install Instructions + +This guide replaces the old `docs/install/` folder. It shows how to clone the repo and sync skills into your local skills folder without deleting your existing custom skills. + +## Quick Start (Project-Local) + +Run from your project root. This pulls the skills into `./skills` without leaving a clone behind. + +### macOS / Linux + +```bash +tmpdir="$(mktemp -d)" && git clone https://github.com/Ariel-Rodriguez/programming-skills.git "$tmpdir" && mkdir -p ./skills && rsync -a --exclude ".git" --exclude ".DS_Store" "$tmpdir/skills/" ./skills/ && rm -rf "$tmpdir" +``` + +### Windows (PowerShell) + +```powershell +$tmp = New-Item -ItemType Directory -Force -Path ([IO.Path]::Combine($env:TEMP, "programming-skills-" + [Guid]::NewGuid().ToString())); git clone https://github.com/Ariel-Rodriguez/programming-skills.git $tmp.FullName; mkdir .\\skills -Force; robocopy (Join-Path $tmp.FullName "skills") (Join-Path (Get-Location) "skills") /E /XO; Remove-Item -Recurse -Force $tmp.FullName +``` + +## 1) Clone the Repo (HTTP) + +### macOS / Linux + +```bash +git clone https://github.com/Ariel-Rodriguez/programming-skills.git +cd programming-skills +``` + +### Windows (PowerShell) + +```powershell +git clone https://github.com/Ariel-Rodriguez/programming-skills.git +cd programming-skills +``` + +## One-Line Install (Clone + Sync + Cleanup) + +Use these if you just want to pull skills into your local folder and remove the temporary clone afterward. + +### macOS / Linux (single command, sync into current project) + +```bash +tmpdir="$(mktemp -d)" && git clone https://github.com/Ariel-Rodriguez/programming-skills.git "$tmpdir" && mkdir -p ./skills && rsync -a --exclude ".git" --exclude ".DS_Store" "$tmpdir/skills/" ./skills/ && rm -rf "$tmpdir" +``` + +### Windows (PowerShell single command, sync into current project) + +```powershell +$tmp = New-Item -ItemType Directory -Force -Path ([IO.Path]::Combine($env:TEMP, "programming-skills-" + [Guid]::NewGuid().ToString())); git clone https://github.com/Ariel-Rodriguez/programming-skills.git $tmp.FullName; mkdir .\\skills -Force; robocopy (Join-Path $tmp.FullName "skills") (Join-Path (Get-Location) "skills") /E /XO; Remove-Item -Recurse -Force $tmp.FullName +``` + +## 2) Sync Skills to Your Project Folder (Recommended) + +Default destination is the current project’s `./skills/` folder. + +If you are already inside the cloned repo, the `skills/` folder is already present and no extra sync is required. + +If you want to copy skills into a different project, run these from the **cloned repo** and set a destination path. + +### macOS / Linux (rsync) + +```bash +DEST=/path/to/your/project/skills +mkdir -p "$DEST" +rsync -a --exclude ".git" --exclude ".DS_Store" skills/ "$DEST/" +``` + +This merges new skills and updates existing ones, without removing your own custom folders. + +### Windows (PowerShell + robocopy) + +```powershell +$dest = "C:\\path\\to\\your\\project\\skills" +mkdir $dest -Force +robocopy .\\skills $dest /E /XO +``` + +`/E` copies subdirectories; `/XO` skips older destination files. + +## 3) Keep Skills Up to Date + +Update your local clone, then re-run the sync step (if you are copying into another project): + +```bash +git pull +rsync -a --exclude ".git" --exclude ".DS_Store" skills/ ./skills/ +``` + +--- + +Skill directory reference: + +``` +https://github.com/Ariel-Rodriguez/programming-skills/tree/main/skills +``` + +## Global Skills Folder (Optional) + +If your AI tool supports a global skills directory, you can sync there instead. Check the tool’s documentation (e.g., Gemini, Claude, Codex) for the exact path and configuration options. diff --git a/docs/install/antigravity.md b/docs/install/antigravity.md deleted file mode 100644 index f3e35df..0000000 --- a/docs/install/antigravity.md +++ /dev/null @@ -1,17 +0,0 @@ -# Installing on Antigravity - -## Automatic Installation (Linux/macOS) - -Run this command in your project root to download the standard skills to `.cursor/skills` (Antigravity is compatible with Cursor skill structure): - -```bash -curl -L https://github.com/Ariel-Rodriguez/programming-skills/releases/latest/download/cursor-antigravity.zip -o skills.zip && unzip skills.zip -d .cursor/skills && rm skills.zip -``` - -## Manual Installation - -1. Create a `.cursor/skills` directory in your project root. -2. Copy the desired skill folders (e.g., `ps-composition-over-coordination`) from this repository into `.cursor/skills`. -3. Configure your agent settings to reference these skills. - -See [AI Prompt Wrapper](../ai-prompt-wrapper.md) for configuration details. diff --git a/docs/install/claude.md b/docs/install/claude.md deleted file mode 100644 index 218bb96..0000000 --- a/docs/install/claude.md +++ /dev/null @@ -1,11 +0,0 @@ -# Installing on Claude - -## Manual Installation - -1. Navigate to your **Project Settings** or **System Prompt** configuration in Claude. -2. Copy the contents of the relevant skills you wish to enforce (from `skills/ps-*/SKILL.md`). -3. specific instructions from [AI Prompt Wrapper](../ai-prompt-wrapper.md) into your project's custom instructions or system prompt. - -## Recommended Setup - -Create a `CLAUDE.md` or `.prompt.md` file in your project root containing the core principles and skill definitions so they are always available in your context. diff --git a/docs/install/copilot.md b/docs/install/copilot.md deleted file mode 100644 index 14cfdcf..0000000 --- a/docs/install/copilot.md +++ /dev/null @@ -1,20 +0,0 @@ -# Installing on GitHub Copilot - -## Configuration - -GitHub Copilot (Enterprise) allows for repository-specific instructions. - -1. Create or edit `.github/copilot/instructions.md` in your repository. -2. Add the core principles defined in the [AI Prompt Wrapper](../ai-prompt-wrapper.md). -3. (Optional) Append the full text of specific critical skills if you want strict enforcement on those topics. - -## Example `instructions.md` - -```markdown -You are an AI assistant giving programming advice. - -When generating code, you must adhere to the following principles: -- **Functional Core / Imperative Shell**: Isolate pure logic. -- **Explicit State Invariants**: Document state validity. -... [Add specific skill instructions here] -``` diff --git a/docs/install/cursor.md b/docs/install/cursor.md deleted file mode 100644 index 6c02c29..0000000 --- a/docs/install/cursor.md +++ /dev/null @@ -1,17 +0,0 @@ -# Installing on Cursor - -## Automatic Installation (Linux/macOS) - -Run this command in your project root to download the standard skills to `.cursor/skills`: - -```bash -curl -L https://github.com/Ariel-Rodriguez/programming-skills/releases/latest/download/cursor-antigravity.zip -o skills.zip && unzip skills.zip -d .cursor/skills && rm skills.zip -``` - -## Manual Installation - -1. Create a `.cursor/skills` directory in your project root. -2. Copy the desired skill folders (e.g., `ps-composition-over-coordination`) from this repository into `.cursor/skills`. -3. Configure your `.cursorrules` to reference these skills. - -See [AI Prompt Wrapper](../ai-prompt-wrapper.md) for configuration details. diff --git a/docs/specs/architecture.md b/docs/specs/architecture.md new file mode 100644 index 0000000..03e3d09 --- /dev/null +++ b/docs/specs/architecture.md @@ -0,0 +1,74 @@ +# Architecture + +## Design Principles + +- **Simplicity**: Skills are defined once with clear principles and examples. +- **Single Source of Truth**: The `skills/` directory contains all skill definitions and their associated benchmark scenarios. +- **Language Independence**: Principles apply to any programming language, with pseudocode for illustrations. +- **Severity-Based Enforcement**: Skills are categorized by impact (BLOCK, WARN, SUGGEST). +- **Reproducible Benchmarks**: Benchmark history is versioned under `tests/data-history/`. + +## Directory Structure + +``` +programming-skills/ +├── skills/ # Skill definitions +│ ├── ps-functional-core-imperative-shell/ +│ │ ├── SKILL.md # Principle & Documentation +│ │ ├── test.json # Benchmark scenarios & expectations +│ │ └── example_case.js # (Optional) External input files +│ └── ... +├── tests/ # Benchmarking suite +│ ├── evaluator.py # Main evaluation engine +│ ├── data-history/ # Versioned benchmark history +│ │ ├── summary---.json +│ │ └── /-.json +│ └── config.yaml # Benchmarking configuration +├── src/ +│ └── pages/ +│ └── benchmarks/ # Dashboard source (index.html + app.js) +├── site/ +│ └── benchmarks/ # Generated output (not committed) +├── ci/ # CI scripts and publishing +└── docs/ # Documentation + ├── specs/ + │ ├── architecture.md + │ ├── benchmark-workflow.md + │ └── benchmark-page.md + ├── contributing.md + ├── install-instructions.md + └── generate-benchmark.md +``` + +## Skill Format + +Each skill is a directory containing: + +1. **SKILL.md**: YAML frontmatter with `name`, `description`, and `severity`. +2. **test.json**: Benchmark test cases for the evaluator. +3. **Support Files**: Optional source files referenced by `test.json`. + +### Severity Levels + +- **BLOCK**: Architectural violations requiring immediate refactoring. +- **WARN**: Significant issues that should be flagged with justification. +- **SUGGEST**: Optional improvements that enhance quality. + +## Benchmarking Flow + +1. **Evaluator** runs baseline + skill prompt per test case. +2. **Summary** written to `tests/data-history/summary---.json`. +3. **Per-skill history** written to `tests/data-history//-.json`. +4. **Dashboard build** reads history and produces: + - `site/benchmarks/benchmarks.json` + - `site/benchmarks/data//data.json` +5. **Publish** copies `site/benchmarks/` into the orphan branch root for GitHub Pages. + +## Platform Support + +Skills are filesystem-based and should be placed where your AI tool reads skills from. See: + +``` +https://agentskills.io/integrate-skills +https://agentskills.io/llms.txt +``` diff --git a/docs/specs/benchmark-page.md b/docs/specs/benchmark-page.md index db0d2ef..e1b5e81 100644 --- a/docs/specs/benchmark-page.md +++ b/docs/specs/benchmark-page.md @@ -9,103 +9,162 @@ This spec describes the HTML/CSS/JS structure for the benchmark dashboard page. - **HTML5**: Semantic structure - **Bootstrap 5**: CSS framework for layout and components - **Vanilla JavaScript**: No build step, pure client-side +- **Prism.js (CDN)**: For syntax highlighting in code blocks - **Chart.js (CDN)**: For future time series charts - **Diff Match Patch (CDN)**: For code comparison highlighting -## Page Structure +## Directory Structure + +``` +src/ +└── pages/ + └── benchmarks/ + ├── index.html # Dashboard template + ├── app.js # Dashboard logic + └── data/ # Optional dev data + +site/ +└── benchmarks/ + ├── index.html # Copied from src/pages/benchmarks + ├── app.js # Copied from src/pages/benchmarks + ├── benchmarks.json # Aggregated data for JS + └── data/ + └── {benchmark_id}/data.json # Per-run data +``` + +## Page Structure (index.html) ```html - + + + + Programming Skills Benchmarks + - + + + + + - +
-
+
+ +
+
+ + +
+
-
-
-
Total Benchmarks
-

0

-
-
+ +
-
-
-
Skills Tested
-

0

-
-
+ +
-
-
-
-
Last Updated
-

-

-
-
+
+ + +
+
+ +
- +
-
-
-
- Filter by: - - +
+
+
Benchmark Results
+
+
+
+ + + + + + + + + + + + + + + +
SkillProviderModelBeforeAfterImprovementDetails
- -
- - - - - - - - - - - - - - - -
SkillProviderModelBeforeAfterImprovementView Details
-
-