From 405a3d0926d38eee24e3f385085d30b4f3da430c Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 14 Apr 2026 12:25:04 +0200 Subject: [PATCH 1/4] Add skill-validator workflow --- .../workflows/skill-validation-comment.yml | 172 ++++++++++++++++++ .github/workflows/skill-validation.yml | 146 +++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 .github/workflows/skill-validation-comment.yml create mode 100644 .github/workflows/skill-validation.yml diff --git a/.github/workflows/skill-validation-comment.yml b/.github/workflows/skill-validation-comment.yml new file mode 100644 index 00000000000..9efec245322 --- /dev/null +++ b/.github/workflows/skill-validation-comment.yml @@ -0,0 +1,172 @@ +name: Skill Validation — PR Comment + +# Posts results from the "Skill Validation" workflow. +# Runs with write permissions but never checks out PR code, +# so it is safe for fork PRs. + +on: + workflow_run: + workflows: ["Skill Validation"] + types: [completed] + +permissions: + pull-requests: write + actions: read # needed to download artifacts + +jobs: + comment: + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' + steps: + - name: Download results artifact + uses: actions/download-artifact@v4 + with: + name: skill-validator-results + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Post PR comment with results + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + const total = parseInt(fs.readFileSync('total.txt', 'utf8').trim(), 10); + if (total === 0) { + console.log('No skills/agents were checked — skipping comment.'); + return; + } + + const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim(), 10); + if (!prNumber) { + console.log('No PR number found — skipping comment.'); + return; + } + + const exitCode = fs.readFileSync('exit-code.txt', 'utf8').trim(); + const skillCount = parseInt(fs.readFileSync('skill-count.txt', 'utf8').trim(), 10); + const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10); + const totalChecked = skillCount + agentCount; + + const marker = ''; + const rawOutput = fs.existsSync('sv-output.txt') + ? fs.readFileSync('sv-output.txt', 'utf8') + : ''; + const output = rawOutput.replace(/\x1b\[[0-9;]*m/g, '').trim(); + + const errorCount = (output.match(/❌/g) || []).length; + const warningCount = (output.match(/⚠/g) || []).length; + const advisoryCount = (output.match(/ℹ/g) || []).length; + + let verdict = '✅ All checks passed'; + if (exitCode !== '0' || errorCount > 0) { + verdict = '⛔ Findings need attention'; + } else if (warningCount > 0 || advisoryCount > 0) { + verdict = '⚠️ Warnings or advisories found'; + } + + const highlightedLines = output + .split('\n') + .map(line => line.trim()) + .filter(Boolean) + .filter(line => !line.startsWith('###')) + .filter(line => /^[❌⚠ℹ]/.test(line)); + + const summaryLines = highlightedLines.length > 0 + ? highlightedLines.slice(0, 10) + : output + .split('\n') + .map(line => line.trim()) + .filter(Boolean) + .filter(line => !line.startsWith('###')) + .slice(0, 10); + + const scopeTable = [ + '| Scope | Checked |', + '|---|---:|', + `| Skills | ${skillCount} |`, + `| Agents | ${agentCount} |`, + `| Total | ${totalChecked} |`, + ]; + + const severityTable = [ + '| Severity | Count |', + '|---|---:|', + `| ❌ Errors | ${errorCount} |`, + `| ⚠️ Warnings | ${warningCount} |`, + `| ℹ️ Advisories | ${advisoryCount} |`, + ]; + + const findingsTable = summaryLines.length === 0 + ? ['_No findings were emitted by the validator._'] + : [ + '| Level | Finding |', + '|---|---|', + ...summaryLines.map(line => { + const level = line.startsWith('❌') + ? '❌' + : line.startsWith('⚠') + ? '⚠️' + : line.startsWith('ℹ') + ? 'ℹ️' + : (exitCode !== '0' ? '⛔' : 'ℹ️'); + const text = line.replace(/^[❌⚠ℹ️\s]+/, '').replace(/\|/g, '\\|'); + return `| ${level} | ${text} |`; + }), + ]; + + const body = [ + marker, + '## 🔍 Skill Validator Results', + '', + `**${verdict}**`, + '', + ...scopeTable, + '', + ...severityTable, + '', + '### Summary', + '', + ...findingsTable, + '', + '
', + 'Full validator output', + '', + '```text', + output || 'No validator output captured.', + '```', + '', + '
', + '', + exitCode !== '0' + ? '> **Note:** The validator returned a non-zero exit code. Please review the findings above before merge.' + : '', + ].filter(Boolean).join('\n'); + + // Find existing comment with our marker + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + }); + + const existing = comments.find(c => c.body.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + console.log(`Updated existing comment ${existing.id}`); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + console.log('Created new PR comment'); + } diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml new file mode 100644 index 00000000000..2cf7b569696 --- /dev/null +++ b/.github/workflows/skill-validation.yml @@ -0,0 +1,146 @@ +name: Skill Validation + +on: + pull_request: + paths: + - '.github/skills/**' + - '.github/agents/**' + - '.github/workflows/skill-validation.yml' + - '.github/workflows/skill-validation-comment.yml' + push: + branches: [main] + paths: + - '.github/skills/**' + - '.github/agents/**' + - '.github/workflows/skill-validation.yml' + - '.github/workflows/skill-validation-comment.yml' + workflow_dispatch: + +concurrency: + group: skill-validation-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + name: Validate skills and agents + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/skills + .github/agents + persist-credentials: false + + # ── Download & cache skill-validator ────────────────────────── + - name: Get cache key date + id: cache-date + run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" + + - name: Restore skill-validator from cache + id: cache-sv + uses: actions/cache/restore@v4 + with: + path: skill-validator-bin + key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + restore-keys: | + skill-validator-linux-x64- + + - name: Download skill-validator + if: steps.cache-sv.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p skill-validator-bin + curl -fsSL --retry 3 --retry-all-errors -o skill-validator.tar.gz \ + https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz + tar -xzf skill-validator.tar.gz -C skill-validator-bin + if [ ! -f skill-validator-bin/skill-validator ]; then + echo "::error::skill-validator binary not found after extraction" + exit 1 + fi + chmod +x skill-validator-bin/skill-validator + + - name: Save skill-validator to cache + if: steps.cache-sv.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: skill-validator-bin + key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + + # ── Run skill-validator check ───────────────────────────────── + - name: Run skill-validator check + id: check + shell: bash + run: | + rc=0 + + if [ -d .github/skills ]; then + echo "::group::Validate skills" + set +e + skill-validator-bin/skill-validator check --skills .github/skills --verbose 2>&1 | tee skill-check-skills.txt + skills_rc=${PIPESTATUS[0]} + set -e + echo "::endgroup::" + if [ "$skills_rc" -ne 0 ]; then rc=1; fi + fi + + if [ -d .github/agents ]; then + echo "::group::Validate agents" + set +e + skill-validator-bin/skill-validator check --agents .github/agents --verbose 2>&1 | tee skill-check-agents.txt + agents_rc=${PIPESTATUS[0]} + set -e + echo "::endgroup::" + if [ "$agents_rc" -ne 0 ]; then rc=1; fi + fi + + # Combine output for the commenting workflow + cat skill-check-skills.txt skill-check-agents.txt > sv-output.txt 2>/dev/null || true + + echo "exit_code=$rc" >> "$GITHUB_OUTPUT" + + # Write to step summary + { + echo "## skill-validator check" + echo "" + if [ "$rc" -eq 0 ]; then + echo "All checks passed." + else + for f in skill-check-skills.txt skill-check-agents.txt; do + if [ -f "$f" ]; then + echo "### ${f}" + echo '```' + head -n 200 "$f" + echo '```' + echo "" + fi + done + fi + } >> "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # ── Upload results for the commenting workflow ──────────────── + - name: Save metadata + if: always() + run: | + mkdir -p sv-results + echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt + echo "13" > sv-results/skill-count.txt + echo "1" > sv-results/agent-count.txt + echo "14" > sv-results/total.txt + echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt + if [ -f sv-output.txt ]; then + cp sv-output.txt sv-results/sv-output.txt + fi + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: skill-validator-results + path: sv-results/ + retention-days: 1 From 777559776f0314511b4e70d2fb7256b4767cd337 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 14 Apr 2026 17:35:27 +0200 Subject: [PATCH 2/4] Fix skills locations/namings --- .github/agents/{expert-reviewer.md => expert-reviewer.agent.md} | 0 .../SKILL.md | 0 .../SKILL.md | 0 .github/skills/changewaves/SKILL.md | 2 +- .../SKILL.md | 0 .../SKILL.md | 0 .../SKILL.md | 0 .github/workflows/shared/review-shared.md | 2 +- 8 files changed, 2 insertions(+), 2 deletions(-) rename .github/agents/{expert-reviewer.md => expert-reviewer.agent.md} (100%) rename .github/skills/{backwards-compatibility => assessing-breaking-changes}/SKILL.md (100%) rename .github/skills/{error-and-warning-authoring => authoring-errors-and-warnings}/SKILL.md (100%) rename .github/skills/{sdk-msbuild-integration => integrating-sdk-and-msbuild}/SKILL.md (100%) rename .github/skills/{binary-log-considerations => maintaining-binary-log-compatibility}/SKILL.md (100%) rename .github/skills/{msbuild-performance => optimizing-msbuild-performance}/SKILL.md (100%) diff --git a/.github/agents/expert-reviewer.md b/.github/agents/expert-reviewer.agent.md similarity index 100% rename from .github/agents/expert-reviewer.md rename to .github/agents/expert-reviewer.agent.md diff --git a/.github/skills/backwards-compatibility/SKILL.md b/.github/skills/assessing-breaking-changes/SKILL.md similarity index 100% rename from .github/skills/backwards-compatibility/SKILL.md rename to .github/skills/assessing-breaking-changes/SKILL.md diff --git a/.github/skills/error-and-warning-authoring/SKILL.md b/.github/skills/authoring-errors-and-warnings/SKILL.md similarity index 100% rename from .github/skills/error-and-warning-authoring/SKILL.md rename to .github/skills/authoring-errors-and-warnings/SKILL.md diff --git a/.github/skills/changewaves/SKILL.md b/.github/skills/changewaves/SKILL.md index a159646d565..5b35e4ec11c 100644 --- a/.github/skills/changewaves/SKILL.md +++ b/.github/skills/changewaves/SKILL.md @@ -8,7 +8,7 @@ argument-hint: 'Add, query, or remove changewaves and changewave checks.' A Change Wave is an opt-out flag that groups risky features together. Users disable features by setting the environment variable `MSBUILDDISABLEFEATURESFROMVERSION` to the wave version. This skill covers the **how** — the full lifecycle: creating a wave, conditioning code on it, testing, documenting, and retiring. -For the **when** — deciding whether a change is a breaking change and whether it needs a ChangeWave at all — see [assessing-breaking-changes](../backwards-compatibility/SKILL.md). +For the **when** — deciding whether a change is a breaking change and whether it needs a ChangeWave at all — see [assessing-breaking-changes](../assessing-breaking-changes/SKILL.md). ## Decide Whether a Change Wave Is Appropriate diff --git a/.github/skills/sdk-msbuild-integration/SKILL.md b/.github/skills/integrating-sdk-and-msbuild/SKILL.md similarity index 100% rename from .github/skills/sdk-msbuild-integration/SKILL.md rename to .github/skills/integrating-sdk-and-msbuild/SKILL.md diff --git a/.github/skills/binary-log-considerations/SKILL.md b/.github/skills/maintaining-binary-log-compatibility/SKILL.md similarity index 100% rename from .github/skills/binary-log-considerations/SKILL.md rename to .github/skills/maintaining-binary-log-compatibility/SKILL.md diff --git a/.github/skills/msbuild-performance/SKILL.md b/.github/skills/optimizing-msbuild-performance/SKILL.md similarity index 100% rename from .github/skills/msbuild-performance/SKILL.md rename to .github/skills/optimizing-msbuild-performance/SKILL.md diff --git a/.github/workflows/shared/review-shared.md b/.github/workflows/shared/review-shared.md index acb36e0f800..b89ea1a8470 100644 --- a/.github/workflows/shared/review-shared.md +++ b/.github/workflows/shared/review-shared.md @@ -29,7 +29,7 @@ safe-outputs: # Expert Code Review -Review pull request #${{ github.event.pull_request.number || github.event.issue.number }} using the `expert-reviewer` agent defined at `.github/agents/expert-reviewer.md`. +Review pull request #${{ github.event.pull_request.number || github.event.issue.number }} using the `expert-reviewer` agent defined at `.github/agents/expert-reviewer.agent.md`. ## Instructions From 49c8c27ea6682f23dcf9089426c9ecb5a4336312 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 14 Apr 2026 17:43:47 +0200 Subject: [PATCH 3/4] Allow repo traversal from the skills --- .github/workflows/skill-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml index 2cf7b569696..2b2296189ff 100644 --- a/.github/workflows/skill-validation.yml +++ b/.github/workflows/skill-validation.yml @@ -81,7 +81,7 @@ jobs: if [ -d .github/skills ]; then echo "::group::Validate skills" set +e - skill-validator-bin/skill-validator check --skills .github/skills --verbose 2>&1 | tee skill-check-skills.txt + skill-validator-bin/skill-validator check --skills .github/skills --allow-repo-traversal --verbose 2>&1 | tee skill-check-skills.txt skills_rc=${PIPESTATUS[0]} set -e echo "::endgroup::" From e7a453de1b9195e9b243d76f068251e12bc984c2 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 14 Apr 2026 18:12:34 +0200 Subject: [PATCH 4/4] Address review feedback: dynamic counts, issues:write, artifact error handling, success summary --- .github/workflows/skill-validation-comment.yml | 8 ++++++++ .github/workflows/skill-validation.yml | 13 ++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/skill-validation-comment.yml b/.github/workflows/skill-validation-comment.yml index 9efec245322..304caba047f 100644 --- a/.github/workflows/skill-validation-comment.yml +++ b/.github/workflows/skill-validation-comment.yml @@ -11,6 +11,7 @@ on: permissions: pull-requests: write + issues: write actions: read # needed to download artifacts jobs: @@ -19,13 +20,20 @@ jobs: if: github.event.workflow_run.event == 'pull_request' steps: - name: Download results artifact + id: download uses: actions/download-artifact@v4 + continue-on-error: true with: name: skill-validator-results run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} + - name: Skip if no artifact + if: steps.download.outcome == 'failure' + run: echo "No artifact found (workflow may have been cancelled). Skipping comment." + - name: Post PR comment with results + if: steps.download.outcome == 'success' uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml index 2b2296189ff..c6934f87e18 100644 --- a/.github/workflows/skill-validation.yml +++ b/.github/workflows/skill-validation.yml @@ -109,6 +109,10 @@ jobs: echo "" if [ "$rc" -eq 0 ]; then echo "All checks passed." + echo "" + skill_count=$(find .github/skills -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) + agent_count=$(find .github/agents -name '*.agent.md' 2>/dev/null | wc -l) + echo "Validated **${skill_count}** skill(s) and **${agent_count}** agent(s)." else for f in skill-check-skills.txt skill-check-agents.txt; do if [ -f "$f" ]; then @@ -128,10 +132,13 @@ jobs: if: always() run: | mkdir -p sv-results + skill_count=$(find .github/skills -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) + agent_count=$(find .github/agents -name '*.agent.md' 2>/dev/null | wc -l) + total=$((skill_count + agent_count)) echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt - echo "13" > sv-results/skill-count.txt - echo "1" > sv-results/agent-count.txt - echo "14" > sv-results/total.txt + echo "$skill_count" > sv-results/skill-count.txt + echo "$agent_count" > sv-results/agent-count.txt + echo "$total" > sv-results/total.txt echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt if [ -f sv-output.txt ]; then cp sv-output.txt sv-results/sv-output.txt