chore: Test Ollama Cloud (#3)

* chore: test ci

* chore: test rnj-1

* chore: test rnj-1-cloud

* chore: Better score and judgment.

* chore: fix the failure threshold.

* chore: fix the table report.

* chore: Final judgment.

* refactor(ci): scalable evaluation orchestration

   Applied programming skills to CI infrastructure:
   - Policy-Mechanism Separation: Config drives execution
   - Composition Over Coordination: Small focused scripts
   - Single Responsibility: Each script does one thing
   - Explicit Boundaries: Clear separation of concerns
   - Single Direction Data Flow: Clear pipeline

   Changes:
   - Renamed validate_skills.sh -> orchestrate_evaluations.sh
   - Created modular scripts: detect_changes, run_evaluation, consolidate_results
   - Enhanced matrix_generator.py with validation
   - Updated workflow for parallel execution support
   - Config-driven provider/model matrix

   Usage:
     /test                    # Run all enabled providers
     /test ollama            # Run only ollama models
     /test parallel          # Run in parallel (matrix strategy)

Author: Ariel-Rodriguez

.github/workflows/full-evaluation.yml

@@ -9,31 +9,69 @@ permissions:
   contents: read
 
 jobs:
-  validate:
+  # Job 1: Generate matrix from config
+  prepare:
     runs-on: ubuntu-latest
     if: github.event.issue.pull_request && contains(github.event.comment.body, '/test') && github.actor == 'Ariel-Rodriguez'
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+      filter: ${{ steps.parse.outputs.filter }}
+      use-parallel: ${{ steps.parse.outputs.use-parallel }}
 
     steps:
       - name: Parse test command
         id: parse
         run: |
           COMMENT="${{ github.event.comment.body }}"
+          FILTER="all"
+          USE_PARALLEL="false"
           
-          # Default provider
-          PROVIDER="ollama"
-          
-          # Check for specific provider
+          # Check for specific provider filter
           if echo "$COMMENT" | grep -q "/test copilot"; then
-            PROVIDER="copilot"
+            FILTER="copilot"
           elif echo "$COMMENT" | grep -q "/test ollama"; then
-            PROVIDER="ollama"
+            FILTER="ollama"
+          elif echo "$COMMENT" | grep -q "/test gemini"; then
+            FILTER="gemini"
           elif echo "$COMMENT" | grep -q "/test all"; then
-            PROVIDER="all"
+            FILTER="all"
+          fi
+          
+          # Check for parallel flag
+          if echo "$COMMENT" | grep -q "parallel"; then
+            USE_PARALLEL="true"
           fi
           
-          echo "provider=$PROVIDER" >> $GITHUB_OUTPUT
-          echo "Testing with provider: $PROVIDER"
+          echo "filter=$FILTER" >> $GITHUB_OUTPUT
+          echo "use-parallel=$USE_PARALLEL" >> $GITHUB_OUTPUT
+          echo "Testing with filter: $FILTER (parallel: $USE_PARALLEL)"
       
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install PyYAML
+        run: pip install pyyaml
+
+      - name: Generate matrix
+        id: generate-matrix
+        run: |
+          MATRIX=$(python3 ci/matrix_generator.py --filter-provider "${{ steps.parse.outputs.filter }}")
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
+  # Job 2: Run evaluations (parallel or sequential based on parse)
+  evaluate:
+    needs: prepare
+    runs-on: ubuntu-latest
+    if: needs.prepare.outputs.use-parallel == 'false'
+    
+    steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
@@ -55,18 +93,103 @@ jobs:
       - name: Set up Python
         run: uv python install 3.11
 
-      - name: Run Skill Validation
+      - name: Run orchestration
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COPILOT_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
         run: |
-          chmod +x ci/validate_skills.sh
-          ./ci/validate_skills.sh ${{ github.event.issue.number }} 50 ${{ steps.parse.outputs.provider }}
+          chmod +x ci/*.sh
+          ./ci/orchestrate_evaluations.sh ${{ github.event.issue.number }} --filter-provider "${{ needs.prepare.outputs.filter }}"
 
-      - name: Post Benchmark Results
+      - name: Post results
         uses: marocchino/sticky-pull-request-comment@v2
         if: always() && hashFiles('comment.md') != ''
         with:
           number: ${{ github.event.issue.number }}
           path: comment.md
+
+  # Job 3: Run evaluations in parallel (matrix strategy)
+  evaluate-parallel:
+    needs: prepare
+    runs-on: ubuntu-latest
+    if: needs.prepare.outputs.use-parallel == 'true'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ github.event.issue.number }}/head
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install Copilot CLI
+        if: matrix.provider == 'copilot'
+        run: npm install -g @github/copilot
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Detect changes
+        id: changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          chmod +x ci/detect_changes.sh
+          MODIFIED_SKILLS=$(./ci/detect_changes.sh ${{ github.event.issue.number }})
+          echo "modified_skills=$MODIFIED_SKILLS" >> $GITHUB_OUTPUT
+
+      - name: Run evaluation for ${{ matrix.display_name }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COPILOT_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
+          MODIFIED_SKILLS: ${{ steps.changes.outputs.modified_skills }}
+        run: |
+          chmod +x ci/run_evaluation.sh
+          ./ci/run_evaluation.sh "${{ matrix.provider }}" "${{ matrix.model }}" 50 "${{ matrix.extra_args }}"
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: results-${{ matrix.provider }}-${{ matrix.model }}
+          path: tests/results/${{ matrix.provider }}/${{ matrix.model }}/
+
+  # Job 4: Consolidate parallel results
+  consolidate:
+    needs: [prepare, evaluate-parallel]
+    runs-on: ubuntu-latest
+    if: needs.prepare.outputs.use-parallel == 'true' && always()
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: tests/results/
+
+      - name: Consolidate results
+        run: |
+          chmod +x ci/consolidate_results.sh
+          ./ci/consolidate_results.sh || true
+
+      - name: Post consolidated results
+        uses: marocchino/sticky-pull-request-comment@v2
+        if: hashFiles('comment.md') != ''
+        with:
+          number: ${{ github.event.issue.number }}
+          path: comment.md

.github/workflows/skill-validation.yml

@@ -0,0 +1,23 @@
+# CI Configuration for Skill Validation
+# Enable/disable providers and specify models to run in parallel matrix.
+
+ollama:
+  enabled: true
+  local: false
+  context: 32000
+  models:
+    - gpt-oss:20b-cloud
+    - rnj-1:8b-cloud
+    - devstral-small-2:24b-cloud
+
+copilot:
+  enabled: false
+  models:
+    - claude-3-5-sonnet
+    - claude-3-haiku
+
+gemini:
+  enabled: false
+  models:
+    - gemini-1.5-flash
+    - gemini-1.5-pro

ci/collect-artifacts.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -e
+
+# consolidate_results.sh
+# Consolidates results from multiple parallel evaluation runs.
+# Composition Over Coordination: Aggregates outputs from independent units.
+# Single Responsibility: Only consolidates, doesn't evaluate.
+
+RESULTS_BASE="tests/results"
+
+echo "==> Consolidating results from all evaluations"
+
+# Find all result directories
+RESULT_DIRS=$(find "$RESULTS_BASE" -type d -name "*" -mindepth 2)
+
+if [ -z "$RESULT_DIRS" ]; then
+    echo "No results found in $RESULTS_BASE"
+    exit 1
+fi
+
+# Check for failures
+FAILED=0
+SUCCEEDED=0
+ALL_RESULTS=""
+
+echo ""
+echo "Results by provider/model:"
+echo "=========================="
+
+for dir in $RESULT_DIRS; do
+    if [ ! -f "$dir/exit_code" ]; then
+        continue
+    fi
+    
+    EXIT_CODE=$(cat "$dir/exit_code")
+    PROVIDER_MODEL=$(echo "$dir" | sed "s|$RESULTS_BASE/||")
+    
+    if [ "$EXIT_CODE" -eq 0 ]; then
+        echo "✅ $PROVIDER_MODEL - PASSED"
+        SUCCEEDED=$((SUCCEEDED + 1))
+    else
+        echo "❌ $PROVIDER_MODEL - FAILED (exit code: $EXIT_CODE)"
+        FAILED=$((FAILED + 1))
+    fi
+    
+    # Collect summary if exists
+    if [ -f "$dir/summary.json" ]; then
+        ALL_RESULTS="$ALL_RESULTS\n### Results: $PROVIDER_MODEL\n"
+        ALL_RESULTS="$ALL_RESULTS$(cat "$dir/summary.json")\n"
+    fi
+done
+
+echo ""
+echo "Summary:"
+echo "========"
+echo "Succeeded: $SUCCEEDED"
+echo "Failed: $FAILED"
+
+# Generate consolidated comment for PR
+if [ -n "$PR_NUMBER" ] && [ "$FAILED" -eq 0 ]; then
+    echo ""
+    echo "Generating consolidated PR comment..."
+    
+    COMMENT="## 🎉 All Evaluations Passed\n\n"
+    COMMENT="${COMMENT}Tested across $SUCCEEDED provider/model combinations.\n\n"
+    COMMENT="${COMMENT}${ALL_RESULTS}"
+    
+    echo -e "$COMMENT" > comment.md
+    echo "✓ Consolidated comment saved to comment.md"
+fi
+
+# Exit with failure if any evaluation failed
+if [ "$FAILED" -gt 0 ]; then
+    echo ""
+    echo "❌ $FAILED evaluation(s) failed"
+    exit 1
+fi
+
+echo ""
+echo "✅ All evaluations passed"
+exit 0