Skip to content

Commit 1c60024

Browse files
JanKrivanekJanKrivanek
authored
Fix eval workflows (#1228)
* Fix eval workflows * Address review: secure two-phase PR comment & byte-based truncation - skill-check.yml: Revert to pull_request trigger (read-only token). Remove PR comment posting; upload results as artifact instead. - skill-check-comment.yml: New workflow_run-triggered workflow that downloads the artifact and posts/updates the PR comment with write permissions, without ever checking out PR code. - skill-quality-report.yml: Replace character-based truncation with byte-based (Buffer.byteLength) limit. Shrink <details> sections structurally before falling back to hard byte-trim, keeping markdown rendering intact.
1 parent 784f373 commit 1c60024

3 files changed

Lines changed: 174 additions & 78 deletions

File tree

‎.github/workflows/skill-check-comment.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: Skill Validator — PR Comment
2+
3+
# Posts results from the "Skill Validator — PR Gate" workflow.
4+
# Runs with write permissions but never checks out PR code,
5+
# so it is safe for fork PRs.
6+
7+
on:
8+
workflow_run:
9+
workflows: ["Skill Validator — PR Gate"]
10+
types: [completed]
11+
12+
permissions:
13+
pull-requests: write
14+
actions: read # needed to download artifacts
15+
16+
jobs:
17+
comment:
18+
runs-on: ubuntu-latest
19+
if: github.event.workflow_run.event == 'pull_request'
20+
steps:
21+
- name: Download results artifact
22+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
23+
with:
24+
name: skill-validator-results
25+
run-id: ${{ github.event.workflow_run.id }}
26+
github-token: ${{ github.token }}
27+
28+
- name: Post PR comment with results
29+
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
30+
with:
31+
script: |
32+
const fs = require('fs');
33+
34+
const total = parseInt(fs.readFileSync('total.txt', 'utf8').trim(), 10);
35+
if (total === 0) {
36+
console.log('No skills/agents were checked — skipping comment.');
37+
return;
38+
}
39+
40+
const prNumber = parseInt(fs.readFileSync('pr-number.txt', 'utf8').trim(), 10);
41+
const exitCode = fs.readFileSync('exit-code.txt', 'utf8').trim();
42+
const skillCount = parseInt(fs.readFileSync('skill-count.txt', 'utf8').trim(), 10);
43+
const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10);
44+
const totalChecked = skillCount + agentCount;
45+
46+
const marker = '<!-- skill-validator-results -->';
47+
const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
48+
49+
// Count errors, warnings, advisories from output
50+
const errorCount = (output.match(/\bError\b/gi) || []).length;
51+
const warningCount = (output.match(/\bWarning\b/gi) || []).length;
52+
const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
53+
54+
let statusLine;
55+
if (errorCount > 0) {
56+
statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
57+
} else if (warningCount > 0) {
58+
statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
59+
} else {
60+
statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
61+
}
62+
63+
const body = [
64+
marker,
65+
'## 🔍 Skill Validator Results',
66+
'',
67+
statusLine,
68+
'',
69+
'<details>',
70+
'<summary>Full output</summary>',
71+
'',
72+
'```',
73+
output,
74+
'```',
75+
'',
76+
'</details>',
77+
'',
78+
exitCode !== '0'
79+
? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
80+
: '',
81+
].join('\n');
82+
83+
// Find existing comment with our marker
84+
const { data: comments } = await github.rest.issues.listComments({
85+
owner: context.repo.owner,
86+
repo: context.repo.repo,
87+
issue_number: prNumber,
88+
per_page: 100,
89+
});
90+
91+
const existing = comments.find(c => c.body.includes(marker));
92+
93+
if (existing) {
94+
await github.rest.issues.updateComment({
95+
owner: context.repo.owner,
96+
repo: context.repo.repo,
97+
comment_id: existing.id,
98+
body,
99+
});
100+
console.log(`Updated existing comment ${existing.id}`);
101+
} else {
102+
await github.rest.issues.createComment({
103+
owner: context.repo.owner,
104+
repo: context.repo.repo,
105+
issue_number: prNumber,
106+
body,
107+
});
108+
console.log('Created new PR comment');
109+
}

‎.github/workflows/skill-check.yml‎

Lines changed: 20 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ on:
1212

1313
permissions:
1414
contents: read
15-
pull-requests: write
16-
issues: write
1715

1816
jobs:
1917
skill-check:
@@ -135,82 +133,27 @@ jobs:
135133
136134
echo "$OUTPUT"
137135
138-
# ── Post / update PR comment ──────────────────────────────────
139-
- name: Post PR comment with results
140-
if: steps.detect.outputs.total != '0'
141-
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
136+
# ── Upload results for the commenting workflow ────────────────
137+
- name: Save metadata
138+
if: always()
139+
run: |
140+
mkdir -p sv-results
141+
echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt
142+
echo "${{ steps.detect.outputs.total }}" > sv-results/total.txt
143+
echo "${{ steps.detect.outputs.skill_count }}" > sv-results/skill-count.txt
144+
echo "${{ steps.detect.outputs.agent_count }}" > sv-results/agent-count.txt
145+
echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt
146+
if [ -f sv-output.txt ]; then
147+
cp sv-output.txt sv-results/sv-output.txt
148+
fi
149+
150+
- name: Upload results
151+
if: always()
152+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
142153
with:
143-
script: |
144-
const fs = require('fs');
145-
146-
const marker = '<!-- skill-validator-results -->';
147-
const output = fs.readFileSync('sv-output.txt', 'utf8').trim();
148-
const exitCode = '${{ steps.check.outputs.exit_code }}';
149-
const skillCount = parseInt('${{ steps.detect.outputs.skill_count }}', 10);
150-
const agentCount = parseInt('${{ steps.detect.outputs.agent_count }}', 10);
151-
const totalChecked = skillCount + agentCount;
152-
153-
// Count errors, warnings, advisories from output
154-
const errorCount = (output.match(/\bError\b/gi) || []).length;
155-
const warningCount = (output.match(/\bWarning\b/gi) || []).length;
156-
const advisoryCount = (output.match(/\bAdvisory\b/gi) || []).length;
157-
158-
let statusLine;
159-
if (errorCount > 0) {
160-
statusLine = `**${totalChecked} resource(s) checked** | ⛔ ${errorCount} error(s) | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
161-
} else if (warningCount > 0) {
162-
statusLine = `**${totalChecked} resource(s) checked** | ⚠️ ${warningCount} warning(s) | ℹ️ ${advisoryCount} advisory(ies)`;
163-
} else {
164-
statusLine = `**${totalChecked} resource(s) checked** | ✅ All checks passed`;
165-
}
166-
167-
const body = [
168-
marker,
169-
'## 🔍 Skill Validator Results',
170-
'',
171-
statusLine,
172-
'',
173-
'<details>',
174-
'<summary>Full output</summary>',
175-
'',
176-
'```',
177-
output,
178-
'```',
179-
'',
180-
'</details>',
181-
'',
182-
exitCode !== '0'
183-
? '> **Note:** Errors were found. These are currently reported as warnings and do not block merge. Please review and address when possible.'
184-
: '',
185-
].join('\n');
186-
187-
// Find existing comment with our marker
188-
const { data: comments } = await github.rest.issues.listComments({
189-
owner: context.repo.owner,
190-
repo: context.repo.repo,
191-
issue_number: context.issue.number,
192-
per_page: 100,
193-
});
194-
195-
const existing = comments.find(c => c.body.includes(marker));
196-
197-
if (existing) {
198-
await github.rest.issues.updateComment({
199-
owner: context.repo.owner,
200-
repo: context.repo.repo,
201-
comment_id: existing.id,
202-
body,
203-
});
204-
console.log(`Updated existing comment ${existing.id}`);
205-
} else {
206-
await github.rest.issues.createComment({
207-
owner: context.repo.owner,
208-
repo: context.repo.repo,
209-
issue_number: context.issue.number,
210-
body,
211-
});
212-
console.log('Created new PR comment');
213-
}
154+
name: skill-validator-results
155+
path: sv-results/
156+
retention-days: 1
214157

215158
- name: Post skip notice if no skills changed
216159
if: steps.detect.outputs.total == '0'

‎.github/workflows/skill-quality-report.yml‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,51 @@ jobs:
248248
249249
core.setOutput('title', title);
250250
core.setOutput('body_file', 'report-body.md');
251-
fs.writeFileSync('report-body.md', body);
251+
252+
// GitHub Issues/Discussions enforce a body size limit on the
253+
// UTF-8 payload (~65536 bytes). Use byte-based limits and prefer
254+
// shrinking verbose <details> sections to keep markdown valid.
255+
const MAX_BODY_BYTES = 65000; // leave some margin
256+
257+
function shrinkDetailsSections(markdown) {
258+
return markdown.replace(
259+
/<details([\s\S]*?)>[\s\S]*?<\/details>/g,
260+
(match, attrs) => {
261+
const placeholder = '\n<summary>Details truncated</summary>\n\n' +
262+
"> Full output was truncated to fit GitHub's body size limit. " +
263+
'See the workflow run for complete output.\n';
264+
return `<details${attrs}>${placeholder}</details>`;
265+
}
266+
);
267+
}
268+
269+
function trimToByteLimit(str, maxBytes) {
270+
const buf = Buffer.from(str, 'utf8');
271+
if (buf.length <= maxBytes) return str;
272+
// Slice bytes and decode, which safely handles multi-byte chars
273+
return buf.slice(0, maxBytes).toString('utf8').replace(/\uFFFD$/, '');
274+
}
275+
276+
const truncNote = '\n\n> **Note:** Output was truncated to fit GitHub\'s body size limit. See the [workflow run](https://github.com/' + context.repo.owner + '/' + context.repo.repo + '/actions/workflows/skill-quality-report.yml) for full output.\n';
277+
const truncNoteBytes = Buffer.byteLength(truncNote, 'utf8');
278+
279+
let finalBody = body;
280+
281+
if (Buffer.byteLength(finalBody, 'utf8') > MAX_BODY_BYTES) {
282+
// First try: collapse <details> sections to reduce size
283+
finalBody = shrinkDetailsSections(finalBody);
284+
}
285+
286+
if (Buffer.byteLength(finalBody, 'utf8') > MAX_BODY_BYTES) {
287+
// Last resort: hard byte-trim + truncation note
288+
finalBody = trimToByteLimit(finalBody, MAX_BODY_BYTES - truncNoteBytes);
289+
}
290+
291+
if (Buffer.byteLength(finalBody, 'utf8') < Buffer.byteLength(body, 'utf8')) {
292+
finalBody += truncNote;
293+
}
294+
295+
fs.writeFileSync('report-body.md', finalBody);
252296
253297
# ── Create Discussion (preferred) or Issue (fallback) ────────
254298
- name: Create Discussion

0 commit comments

Comments
 (0)