Add static eval for repo skills/agents via skill-validator

.github/skills/changewaves/SKILL.md

@@ -8,7 +8,7 @@ argument-hint: 'Add, query, or remove changewaves and changewave checks.'
 
 A Change Wave is an opt-out flag that groups risky features together. Users disable features by setting the environment variable `MSBUILDDISABLEFEATURESFROMVERSION` to the wave version. This skill covers the **how** — the full lifecycle: creating a wave, conditioning code on it, testing, documenting, and retiring.
 
-For the **when** — deciding whether a change is a breaking change and whether it needs a ChangeWave at all — see [assessing-breaking-changes](../backwards-compatibility/SKILL.md).
+For the **when** — deciding whether a change is a breaking change and whether it needs a ChangeWave at all — see [assessing-breaking-changes](../assessing-breaking-changes/SKILL.md).
 
 
 ## Decide Whether a Change Wave Is Appropriate

.github/workflows/shared/review-shared.md

@@ -29,7 +29,7 @@ safe-outputs:
 
 # Expert Code Review
 
-Review pull request #${{ github.event.pull_request.number || github.event.issue.number }} using the `expert-reviewer` agent defined at `.github/agents/expert-reviewer.md`.
+Review pull request #${{ github.event.pull_request.number || github.event.issue.number }} using the `expert-reviewer` agent defined at `.github/agents/expert-reviewer.agent.md`.
 
 ## Instructions
 

.github/workflows/skill-validation-comment.yml

@@ -0,0 +1,180 @@
+name: Skill Validation — PR Comment
+
+# Posts results from the "Skill Validation" workflow.
+# Runs with write permissions but never checks out PR code,
+# so it is safe for fork PRs.
+
+on:
+  workflow_run:
+    workflows: ["Skill Validation"]
+    types: [completed]
+
+permissions:
+  pull-requests: write
+  issues: write
+  actions: read # needed to download artifacts
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download results artifact
+        id: download
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          name: skill-validator-results
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+
+      - name: Skip if no artifact
+        if: steps.download.outcome == 'failure'
+        run: echo "No artifact found (workflow may have been cancelled). Skipping comment."
+
+      - name: Post PR comment with results
+        if: steps.download.outcome == 'success'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            const total = parseInt(fs.readFileSync('total.txt', 'utf8').trim(), 10);
+            if (total === 0) {
+              console.log('No skills/agents were checked — skipping comment.');
+              return;
+            }
+
+            const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim(), 10);
+            if (!prNumber) {
+              console.log('No PR number found — skipping comment.');
+              return;
+            }
+
+            const exitCode = fs.readFileSync('exit-code.txt', 'utf8').trim();
+            const skillCount = parseInt(fs.readFileSync('skill-count.txt', 'utf8').trim(), 10);
+            const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10);
+            const totalChecked = skillCount + agentCount;
+
+            const marker = '<!-- skill-validator-results -->';
+            const rawOutput = fs.existsSync('sv-output.txt')
+              ? fs.readFileSync('sv-output.txt', 'utf8')
+              : '';
+            const output = rawOutput.replace(/\x1b\[[0-9;]*m/g, '').trim();
+
+            const errorCount = (output.match(/❌/g) || []).length;
+            const warningCount = (output.match(/⚠/g) || []).length;
+            const advisoryCount = (output.match(/ℹ/g) || []).length;
+
+            let verdict = '✅ All checks passed';
+            if (exitCode !== '0' || errorCount > 0) {
+              verdict = '⛔ Findings need attention';
+            } else if (warningCount > 0 || advisoryCount > 0) {
+              verdict = '⚠️ Warnings or advisories found';
+            }
+
+            const highlightedLines = output
+              .split('\n')
+              .map(line => line.trim())
+              .filter(Boolean)
+              .filter(line => !line.startsWith('###'))
+              .filter(line => /^[❌⚠ℹ]/.test(line));
+
+            const summaryLines = highlightedLines.length > 0
+              ? highlightedLines.slice(0, 10)
+              : output
+                  .split('\n')
+                  .map(line => line.trim())
+                  .filter(Boolean)
+                  .filter(line => !line.startsWith('###'))
+                  .slice(0, 10);
+
+            const scopeTable = [
+              '| Scope | Checked |',
+              '|---|---:|',
+              `| Skills | ${skillCount} |`,
+              `| Agents | ${agentCount} |`,
+              `| Total | ${totalChecked} |`,
+            ];
+
+            const severityTable = [
+              '| Severity | Count |',
+              '|---|---:|',
+              `| ❌ Errors | ${errorCount} |`,
+              `| ⚠️ Warnings | ${warningCount} |`,
+              `| ℹ️ Advisories | ${advisoryCount} |`,
+            ];
+
+            const findingsTable = summaryLines.length === 0
+              ? ['_No findings were emitted by the validator._']
+              : [
+                  '| Level | Finding |',
+                  '|---|---|',
+                  ...summaryLines.map(line => {
+                    const level = line.startsWith('❌')
+                      ? '❌'
+                      : line.startsWith('⚠')
+                        ? '⚠️'
+                        : line.startsWith('ℹ')
+                          ? 'ℹ️'
+                          : (exitCode !== '0' ? '⛔' : 'ℹ️');
+                    const text = line.replace(/^[❌⚠ℹ️\s]+/, '').replace(/\|/g, '\\|');
+                    return `| ${level} | ${text} |`;
+                  }),
+                ];
+
+            const body = [
+              marker,
+              '## 🔍 Skill Validator Results',
+              '',
+              `**${verdict}**`,
+              '',
+              ...scopeTable,
+              '',
+              ...severityTable,
+              '',
+              '### Summary',
+              '',
+              ...findingsTable,
+              '',
+              '<details>',
+              '<summary>Full validator output</summary>',
+              '',
+              '```text',
+              output || 'No validator output captured.',
+              '```',
+              '',
+              '</details>',
+              '',
+              exitCode !== '0'
+                ? '> **Note:** The validator returned a non-zero exit code. Please review the findings above before merge.'
+                : '',
+            ].filter(Boolean).join('\n');
+
+            // Find existing comment with our marker
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            });
+
+            const existing = comments.find(c => c.body.includes(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              console.log(`Updated existing comment ${existing.id}`);
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+              console.log('Created new PR comment');
+            }

.github/workflows/skill-validation.yml

@@ -0,0 +1,153 @@
+name: Skill Validation
+
+on:
+  pull_request:
+    paths:
+      - '.github/skills/**'
+      - '.github/agents/**'
+      - '.github/workflows/skill-validation.yml'
+      - '.github/workflows/skill-validation-comment.yml'
+  push:
+    branches: [main]
+    paths:
+      - '.github/skills/**'
+      - '.github/agents/**'
+      - '.github/workflows/skill-validation.yml'
+      - '.github/workflows/skill-validation-comment.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: skill-validation-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: Validate skills and agents
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/skills
+            .github/agents
+          persist-credentials: false
+
+      # ── Download & cache skill-validator ──────────────────────────
+      - name: Get cache key date
+        id: cache-date
+        run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
+
+      - name: Restore skill-validator from cache
+        id: cache-sv
+        uses: actions/cache/restore@v4
+        with:
+          path: skill-validator-bin
+          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+          restore-keys: |
+            skill-validator-linux-x64-
+
+      - name: Download skill-validator
+        if: steps.cache-sv.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          mkdir -p skill-validator-bin
+          curl -fsSL --retry 3 --retry-all-errors -o skill-validator.tar.gz \
+            https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz
+          tar -xzf skill-validator.tar.gz -C skill-validator-bin
+          if [ ! -f skill-validator-bin/skill-validator ]; then
+            echo "::error::skill-validator binary not found after extraction"
+            exit 1
+          fi
+          chmod +x skill-validator-bin/skill-validator
+
+      - name: Save skill-validator to cache
+        if: steps.cache-sv.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: skill-validator-bin
+          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+
+      # ── Run skill-validator check ─────────────────────────────────
+      - name: Run skill-validator check
+        id: check
+        shell: bash
+        run: |
+          rc=0
+
+          if [ -d .github/skills ]; then
+            echo "::group::Validate skills"
+            set +e
+            skill-validator-bin/skill-validator check --skills .github/skills --allow-repo-traversal --verbose 2>&1 | tee skill-check-skills.txt
+            skills_rc=${PIPESTATUS[0]}
+            set -e
+            echo "::endgroup::"
+            if [ "$skills_rc" -ne 0 ]; then rc=1; fi
+          fi
+
+          if [ -d .github/agents ]; then
+            echo "::group::Validate agents"
+            set +e
+            skill-validator-bin/skill-validator check --agents .github/agents --verbose 2>&1 | tee skill-check-agents.txt
+            agents_rc=${PIPESTATUS[0]}
+            set -e
+            echo "::endgroup::"
+            if [ "$agents_rc" -ne 0 ]; then rc=1; fi
+          fi
+
+          # Combine output for the commenting workflow
+          cat skill-check-skills.txt skill-check-agents.txt > sv-output.txt 2>/dev/null || true
+
+          echo "exit_code=$rc" >> "$GITHUB_OUTPUT"
+
+          # Write to step summary
+          {
+            echo "## skill-validator check"
+            echo ""
+            if [ "$rc" -eq 0 ]; then
+              echo "All checks passed."
+              echo ""
+              skill_count=$(find .github/skills -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
+              agent_count=$(find .github/agents -name '*.agent.md' 2>/dev/null | wc -l)
+              echo "Validated **${skill_count}** skill(s) and **${agent_count}** agent(s)."
+            else
+              for f in skill-check-skills.txt skill-check-agents.txt; do
+                if [ -f "$f" ]; then
+                  echo "### ${f}"
+                  echo '```'
+                  head -n 200 "$f"
+                  echo '```'
+                  echo ""
+                fi
+              done
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+          exit "$rc"
+
+      # ── Upload results for the commenting workflow ────────────────
+      - name: Save metadata
+        if: always()
+        run: |
+          mkdir -p sv-results
+          skill_count=$(find .github/skills -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
+          agent_count=$(find .github/agents -name '*.agent.md' 2>/dev/null | wc -l)
+          total=$((skill_count + agent_count))
+          echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt
+          echo "$skill_count" > sv-results/skill-count.txt
+          echo "$agent_count" > sv-results/agent-count.txt
+          echo "$total" > sv-results/total.txt
+          echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt
+          if [ -f sv-output.txt ]; then
+            cp sv-output.txt sv-results/sv-output.txt
+          fi
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: skill-validator-results
+          path: sv-results/
+          retention-days: 1