Skip to content

Commit ce4f04e

Browse files
dakotasanchezdakotasanchez
authored
Add promptfoo eval harness for skills + consolidate MCP endpoint (#65)
1 parent fc69376 commit ce4f04e

37 files changed

Lines changed: 18050 additions & 24 deletions

‎.github/workflows/eval-skills.yml‎

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
name: Skill Evals
2+
3+
# Diff-gated evaluation runner for the public-facing skills under skills/.
4+
#
5+
# Triggers:
6+
# - pull_request: comment score diff vs main; do not commit anything.
7+
# - schedule: nightly run on changed skills, commit refreshed
8+
# eval-scores.json + per-skill README badges back to main.
9+
# - workflow_dispatch: manual full or partial re-run.
10+
#
11+
# Cost shape: only suites whose source has actually changed (per
12+
# evals/scripts/diff-changed-skills.js) get re-evaluated, so a typical PR
13+
# touching one skill costs roughly one suite's worth of API tokens.
14+
15+
on:
16+
pull_request:
17+
paths:
18+
- "skills/**"
19+
- "evals/**"
20+
- ".github/workflows/eval-skills.yml"
21+
schedule:
22+
# 09:17 UTC daily - off the hour to avoid lining up with API rate limits.
23+
- cron: "17 9 * * *"
24+
workflow_dispatch:
25+
inputs:
26+
run_all:
27+
description: "Re-run every suite regardless of diff"
28+
type: boolean
29+
default: false
30+
31+
concurrency:
32+
group: skill-evals-${{ github.ref }}
33+
cancel-in-progress: true
34+
35+
permissions:
36+
# contents: write is needed only on `schedule` / `workflow_dispatch` so the
37+
# aggregate job can push the refreshed eval-scores.json and per-skill README
38+
# badges back to main. Pull requests use the same workflow but the commit
39+
# step is gated on event_name, so PR runs effectively only need read.
40+
contents: write
41+
pull-requests: write
42+
43+
jobs:
44+
unit-test:
45+
name: "Unit tests"
46+
runs-on: ubuntu-latest
47+
steps:
48+
- name: Checkout
49+
uses: actions/checkout@v4
50+
51+
- name: Setup Node.js
52+
uses: actions/setup-node@v4
53+
with:
54+
node-version: "22"
55+
cache: "npm"
56+
cache-dependency-path: evals/package-lock.json
57+
58+
- name: Install eval dependencies
59+
run: npm ci --legacy-peer-deps
60+
working-directory: evals
61+
62+
- name: Run unit tests
63+
run: npm test
64+
working-directory: evals
65+
66+
diff:
67+
name: "Compute changed suites"
68+
runs-on: ubuntu-latest
69+
outputs:
70+
slugs: ${{ steps.compute.outputs.slugs }}
71+
has_changes: ${{ steps.compute.outputs.has_changes }}
72+
steps:
73+
- name: Checkout
74+
uses: actions/checkout@v4
75+
with:
76+
fetch-depth: 0
77+
78+
- name: Compute changed suites
79+
id: compute
80+
run: |
81+
if [[ "${{ inputs.run_all }}" == "true" ]]; then
82+
# run_all overrides: list every suite
83+
slugs="$(node -e 'const m=require("./evals/scripts/_manifest");process.stdout.write(JSON.stringify(m.SUITES.map(s=>s.suite)))')"
84+
else
85+
slugs="$(node evals/scripts/diff-changed-skills.js --json --verbose)"
86+
fi
87+
echo "slugs=${slugs}" >> "$GITHUB_OUTPUT"
88+
if [[ "${slugs}" == "[]" ]]; then
89+
echo "has_changes=false" >> "$GITHUB_OUTPUT"
90+
else
91+
echo "has_changes=true" >> "$GITHUB_OUTPUT"
92+
fi
93+
echo "Changed suites: ${slugs}"
94+
95+
evaluate:
96+
name: "Evaluate ${{ matrix.suite }}"
97+
needs: [unit-test, diff]
98+
if: needs.diff.outputs.has_changes == 'true'
99+
runs-on: ubuntu-latest
100+
strategy:
101+
fail-fast: false
102+
max-parallel: 3
103+
matrix:
104+
suite: ${{ fromJson(needs.diff.outputs.slugs) }}
105+
steps:
106+
- name: Checkout
107+
uses: actions/checkout@v4
108+
with:
109+
fetch-depth: 0
110+
111+
- name: Setup Node.js
112+
uses: actions/setup-node@v4
113+
with:
114+
node-version: "22"
115+
cache: "npm"
116+
cache-dependency-path: evals/package-lock.json
117+
118+
- name: Install eval dependencies
119+
run: npm ci --legacy-peer-deps
120+
working-directory: evals
121+
122+
- name: Run eval suite
123+
env:
124+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
125+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
126+
AGENT_MODEL: ${{ vars.AGENT_MODEL || 'claude-sonnet-4-6' }}
127+
RUBRIC_MODEL: ${{ vars.RUBRIC_MODEL || 'anthropic:messages:claude-haiku-4-5-20251001' }}
128+
run: node scripts/aggregate.js --run --only=${{ matrix.suite }}
129+
working-directory: evals
130+
131+
- name: Upload suite results
132+
if: always()
133+
uses: actions/upload-artifact@v4
134+
with:
135+
name: results-${{ matrix.suite }}
136+
path: evals/${{ matrix.suite }}/results.json
137+
retention-days: 14
138+
139+
aggregate:
140+
name: "Aggregate scores"
141+
needs: [diff, evaluate]
142+
if: needs.diff.outputs.has_changes == 'true' && always() && needs.evaluate.result != 'cancelled'
143+
runs-on: ubuntu-latest
144+
steps:
145+
- name: Checkout
146+
uses: actions/checkout@v4
147+
with:
148+
fetch-depth: 0
149+
token: ${{ secrets.GITHUB_TOKEN }}
150+
151+
- name: Setup Node.js
152+
uses: actions/setup-node@v4
153+
with:
154+
node-version: "22"
155+
cache: "npm"
156+
cache-dependency-path: evals/package-lock.json
157+
158+
- name: Install eval dependencies
159+
run: npm ci --legacy-peer-deps
160+
working-directory: evals
161+
162+
- name: Download all suite results
163+
uses: actions/download-artifact@v4
164+
with:
165+
path: artifact-results
166+
pattern: results-*
167+
merge-multiple: false
168+
169+
- name: Stage suite results into evals/<suite>/results.json
170+
run: |
171+
set -e
172+
shopt -s nullglob
173+
for d in artifact-results/results-*; do
174+
name=$(basename "$d" | sed 's/^results-//')
175+
mkdir -p "evals/$name"
176+
if [[ -f "$d/results.json" ]]; then
177+
cp "$d/results.json" "evals/$name/results.json"
178+
echo "Staged evals/$name/results.json"
179+
fi
180+
done
181+
182+
- name: Save previous eval-scores.json for diff
183+
run: |
184+
if [[ -f eval-scores.json ]]; then
185+
cp eval-scores.json /tmp/eval-scores-before.json
186+
else
187+
echo '{"schemaVersion":1,"updatedAt":null,"skills":{}}' > /tmp/eval-scores-before.json
188+
fi
189+
190+
- name: Aggregate
191+
env:
192+
SUITES_JSON: ${{ needs.diff.outputs.slugs }}
193+
run: |
194+
slugs=$(echo "$SUITES_JSON" | node -e 'let s="";process.stdin.on("data",c=>s+=c);process.stdin.on("end",()=>{const a=JSON.parse(s);process.stdout.write(a.join(","))})')
195+
if [[ -z "$slugs" ]]; then
196+
echo "No suites to aggregate"
197+
exit 0
198+
fi
199+
node scripts/aggregate.js --only="$slugs"
200+
working-directory: evals
201+
202+
- name: Render README badges
203+
if: always()
204+
run: node evals/scripts/render-badges.js
205+
206+
- name: PR comment with score diff
207+
if: always() && github.event_name == 'pull_request'
208+
uses: actions/github-script@v7
209+
env:
210+
BEFORE_PATH: /tmp/eval-scores-before.json
211+
AFTER_PATH: ${{ github.workspace }}/eval-scores.json
212+
with:
213+
script: |
214+
const fs = require('node:fs');
215+
const before = JSON.parse(fs.readFileSync(process.env.BEFORE_PATH, 'utf-8'));
216+
const after = JSON.parse(fs.readFileSync(process.env.AFTER_PATH, 'utf-8'));
217+
const lines = [
218+
'<!-- skill-evals-comment -->',
219+
'## Skill eval results',
220+
'',
221+
'| Skill | Before | After | Δ |',
222+
'|-------|-------:|------:|----:|',
223+
];
224+
const keys = new Set([
225+
...Object.keys(before.skills || {}),
226+
...Object.keys(after.skills || {}),
227+
]);
228+
for (const key of [...keys].sort()) {
229+
const b = (before.skills || {})[key];
230+
const a = (after.skills || {})[key];
231+
if (!a) continue;
232+
const beforeStr = b && b.score !== null ? `${b.score}/100 (${b.passed}/${b.total})` : '-';
233+
const afterStr = a.score !== null ? `${a.score}/100 (${a.passed}/${a.total})` : 'errored';
234+
const delta = (b && b.score !== null && a.score !== null)
235+
? (a.score - b.score === 0 ? 'no change' : (a.score - b.score > 0 ? `+${a.score - b.score}` : `${a.score - b.score}`))
236+
: 'new';
237+
lines.push(`| \`${key}\` | ${beforeStr} | ${afterStr} | ${delta} |`);
238+
}
239+
lines.push('');
240+
lines.push('_Only suites whose source actually changed since their last recorded score were re-run. Soft-failing while we stabilise the baseline._');
241+
const body = lines.join('\n');
242+
243+
const { data: comments } = await github.rest.issues.listComments({
244+
owner: context.repo.owner,
245+
repo: context.repo.repo,
246+
issue_number: context.issue.number,
247+
});
248+
const existing = comments.find((c) => c.body && c.body.startsWith('<!-- skill-evals-comment -->'));
249+
if (existing) {
250+
await github.rest.issues.updateComment({
251+
owner: context.repo.owner,
252+
repo: context.repo.repo,
253+
comment_id: existing.id,
254+
body,
255+
});
256+
} else {
257+
await github.rest.issues.createComment({
258+
owner: context.repo.owner,
259+
repo: context.repo.repo,
260+
issue_number: context.issue.number,
261+
body,
262+
});
263+
}
264+
265+
- name: Commit refreshed scores and badges
266+
if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
267+
run: |
268+
if git diff --quiet eval-scores.json skills/; then
269+
echo "No score or badge changes to commit"
270+
exit 0
271+
fi
272+
git config user.name 'github-actions[bot]'
273+
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
274+
git add eval-scores.json
275+
git add skills/**/README.md
276+
git commit -m "chore(evals): refresh eval-scores.json and README badges"
277+
git push origin HEAD:${{ github.ref_name }}
278+
279+
eval-gate:
280+
name: "Evaluate gate"
281+
needs: [diff, evaluate]
282+
if: always()
283+
runs-on: ubuntu-latest
284+
steps:
285+
- name: Check evaluate results
286+
run: |
287+
if [[ "${{ needs.diff.outputs.has_changes }}" != "true" ]]; then
288+
echo "No suites changed — gate passes"
289+
exit 0
290+
fi
291+
if [[ "${{ needs.evaluate.result }}" == "success" ]]; then
292+
echo "All evaluate jobs passed"
293+
exit 0
294+
fi
295+
echo "One or more evaluate jobs failed (result: ${{ needs.evaluate.result }})"
296+
exit 1

‎.mcp.json‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
11
{
22
"mcpServers": {
3-
"LaunchDarkly Feature Management": {
3+
"LaunchDarkly": {
44
"type": "http",
5-
"url": "https://mcp.launchdarkly.com/mcp/fm",
6-
"headers": {}
7-
},
8-
"LaunchDarkly AI Configs": {
9-
"type": "http",
10-
"url": "https://mcp.launchdarkly.com/mcp/aiconfigs",
5+
"url": "https://mcp.launchdarkly.com/mcp/launchdarkly",
116
"headers": {}
127
}
138
}

‎CONTRIBUTING.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,11 @@ The metrics skills have a Promptfoo suite under `tests/`. Run it before merging
4040

4141
```bash
4242
export ANTHROPIC_API_KEY=sk-ant-...
43-
npm run test:llm-evals
43+
npm run eval:all
4444
```
4545

46-
Run from the repository root, or from `tests/` with `npm run test:llm-evals` (same as `npm run eval`). For a single skill, use `npm run eval:create`, `eval:choose`, or `eval:instrument` inside `tests/`.
46+
Run from the repository root. To run a single suite, `cd evals` and use `npm run eval:<suite-name>` (e.g., `eval:aiconfig-create`). View results with `npm run eval:view`.
47+
4748

4849
## Documentation
4950

0 commit comments

Comments
 (0)