Skip to content

Commit 788ae40

Browse files
JanKrivanekJanKrivanek
committed
Add static eval via skill-validator
1 parent 4c98da8 commit 788ae40

2 files changed

Lines changed: 507 additions & 0 deletions

File tree

‎.github/workflows/skill-check.yml‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
name: Skill Validator — PR Gate
2+
3+
on:
4+
pull_request:
5+
branches: [staged]
6+
types: [opened, synchronize, reopened]
7+
paths:
8+
- "skills/**"
9+
- "agents/**"
10+
- "plugins/**/skills/**"
11+
- "plugins/**/agents/**"
12+
13+
permissions:
14+
contents: read
15+
pull-requests: write
16+
17+
jobs:
18+
skill-check:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- name: Checkout code
22+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
23+
with:
24+
fetch-depth: 0
25+
26+
# ── Download & cache skill-validator ──────────────────────────
27+
- name: Restore skill-validator from cache
28+
id: cache-sv
29+
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
30+
with:
31+
path: .skill-validator
32+
key: skill-validator-linux-x64-nightly
33+
34+
- name: Download skill-validator
35+
if: steps.cache-sv.outputs.cache-hit != 'true'
36+
run: |
37+
mkdir -p .skill-validator
38+
curl -fsSL \
39+
"https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
40+
-o .skill-validator/skill-validator-linux-x64.tar.gz
41+
tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
42+
rm .skill-validator/skill-validator-linux-x64.tar.gz
43+
chmod +x .skill-validator/skill-validator
44+
45+
- name: Save skill-validator to cache
46+
if: steps.cache-sv.outputs.cache-hit != 'true'
47+
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
48+
with:
49+
path: .skill-validator
50+
key: skill-validator-linux-x64-nightly
51+
52+
# ── Detect changed skills & agents ────────────────────────────
53+
- name: Detect changed skills and agents
54+
id: detect
55+
run: |
56+
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
57+
58+
# Extract unique skill directories that were touched
59+
SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^skills/[^/]+' | sort -u || true)
60+
61+
# Extract agent files that were touched
62+
AGENT_FILES=$(echo "$CHANGED_FILES" | grep -oP '^agents/[^/]+\.agent\.md$' | sort -u || true)
63+
64+
# Extract plugin skill directories
65+
PLUGIN_SKILL_DIRS=$(echo "$CHANGED_FILES" | grep -oP '^plugins/[^/]+/skills/[^/]+' | sort -u || true)
66+
67+
# Build CLI arguments for --skills
68+
SKILL_ARGS=""
69+
for dir in $SKILL_DIRS $PLUGIN_SKILL_DIRS; do
70+
if [ -d "$dir" ]; then
71+
SKILL_ARGS="$SKILL_ARGS $dir"
72+
fi
73+
done
74+
75+
# Build CLI arguments for --agents
76+
AGENT_ARGS=""
77+
for f in $AGENT_FILES; do
78+
if [ -f "$f" ]; then
79+
AGENT_ARGS="$AGENT_ARGS $f"
80+
fi
81+
done
82+
83+
SKILL_COUNT=$(echo "$SKILL_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0)
84+
AGENT_COUNT=$(echo "$AGENT_ARGS" | xargs -n1 2>/dev/null | wc -l || echo 0)
85+
TOTAL=$((SKILL_COUNT + AGENT_COUNT))
86+
87+
echo "skill_args=$SKILL_ARGS" >> "$GITHUB_OUTPUT"
88+
echo "agent_args=$AGENT_ARGS" >> "$GITHUB_OUTPUT"
89+
echo "total=$TOTAL" >> "$GITHUB_OUTPUT"
90+
echo "skill_count=$SKILL_COUNT" >> "$GITHUB_OUTPUT"
91+
echo "agent_count=$AGENT_COUNT" >> "$GITHUB_OUTPUT"
92+
93+
echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check."
94+
95+
# ── Run skill-validator check ─────────────────────────────────
96+
- name: Run skill-validator check
97+
id: check
98+
if: steps.detect.outputs.total != '0'
99+
run: |
100+
SKILL_ARGS="${{ steps.detect.outputs.skill_args }}"
101+
AGENT_ARGS="${{ steps.detect.outputs.agent_args }}"
102+
103+
CMD=".skill-validator/skill-validator check --verbose"
104+
105+
if [ -n "$SKILL_ARGS" ]; then
106+
CMD="$CMD --skills $SKILL_ARGS"
107+
fi
108+
109+
if [ -n "$AGENT_ARGS" ]; then
110+
CMD="$CMD --agents $AGENT_ARGS"
111+
fi
112+
113+
echo "Running: $CMD"
114+
115+
# Capture output; don't fail the workflow (warn-only mode)
116+
set +e
117+
OUTPUT=$($CMD 2>&1)
118+
EXIT_CODE=$?
119+
set -e
120+
121+
echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
122+
123+
# Save output to file (multi-line safe)
124+
echo "$OUTPUT" > sv-output.txt
125+
126+
echo "$OUTPUT"
127+
128+
# ── Post / update PR comment ──────────────────────────────────
129+
- name: Post PR comment with results
130+
if: steps.detect.outputs.total != '0'
131+
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
132+
with:
133+
script: |
134+
const fs = require('fs');
135+
136+
const marker = '<!-- skill-validator-results -->';
137+
const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
138+
const exitCode = '${{ steps.check.outputs.exit_code }}';
139+
const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10);
140+
const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10);
141+
const totalChecked = skillCount + agentCount;
142+
143+
// Count errors, warnings, advisories from output
144+
const errorCount = (output.match(/\bError\b/gi) || []).length;
145+
const warningCount = (output.match(/\bWarning\b/gi) || []).length;
146+
const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
147+
148+
let statusLine;
149+
if (errorCount > 0) {
150+
statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
151+
} else if (warningCount > 0) {
152+
statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
153+
} else {
154+
statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
155+
}
156+
157+
const body = [
158+
marker,
159+
'## 🔍 Skill Validator Results',
160+
'',
161+
statusLine,
162+
'',
163+
'<details>',
164+
'<summary>Full output</summary>',
165+
'',
166+
'```',
167+
output,
168+
'```',
169+
'',
170+
'</details>',
171+
'',
172+
exitCode !== '0'
173+
? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
174+
: '',
175+
].join('\n');
176+
177+
// Find existing comment with our marker
178+
const { data: comments } = await github.rest.issues.listComments({
179+
owner: context.repo.owner,
180+
repo: context.repo.repo,
181+
issue_number: context.issue.number,
182+
per_page: 100,
183+
});
184+
185+
const existing = comments.find(c => c.body.includes(marker));
186+
187+
if (existing) {
188+
await github.rest.issues.updateComment({
189+
owner: context.repo.owner,
190+
repo: context.repo.repo,
191+
comment_id: existing.id,
192+
body,
193+
});
194+
console.log(`Updated existing comment ${existing.id}`);
195+
} else {
196+
await github.rest.issues.createComment({
197+
owner: context.repo.owner,
198+
repo: context.repo.repo,
199+
issue_number: context.issue.number,
200+
body,
201+
});
202+
console.log('Created new PR comment');
203+
}
204+
205+
- name: Post skip notice if no skills changed
206+
if: steps.detect.outputs.total == '0'
207+
run: echo "No skill or agent files changed in this PR — skipping validation."

0 commit comments

Comments
 (0)