Skip to content

Commit ebae038

Browse files
committed
fix(learn): show all catalog skills to LLM for better recommendations
The pre-filter in buildLearnUserPrompt only showed skills matching the project's detected languages/frameworks, silently dropping skills with missing metadata or cross-domain relevance. Now all skills are shown (matching first, then others, capped at 30) so the LLM can discover non-obvious connections and rank the full catalog.
1 parent f76fc5e commit ebae038

2 files changed

Lines changed: 80 additions & 24 deletions

File tree

src/skills/learnPrompts.ts

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -101,32 +101,51 @@ export function buildLearnUserPrompt(
101101
const skillWord = registrySkills.length === 1 ? 'community skill' : 'community skills';
102102
parts.push(`${registrySkills.length} ${skillWord} available.`);
103103

104-
// Show only skills that match the project's languages/frameworks
104+
// Show all skills to the LLM so it can discover cross-domain relevance.
105+
// Matching skills are listed first so the LLM prioritizes them.
105106
const projectLanguages = new Set(analysis.languages.map((l) => l.toLowerCase()));
106107
const projectFrameworks = new Set(analysis.frameworks.map((f) => f.toLowerCase()));
107108

108-
const relevant = registrySkills.filter((skill) => {
109+
const matching: typeof registrySkills = [];
110+
const other: typeof registrySkills = [];
111+
for (const skill of registrySkills) {
109112
const skillLangs = (skill.languages ?? []).map((l) => l.toLowerCase());
110113
const skillFw = (skill.frameworks ?? []).map((f) => f.toLowerCase());
111-
return (
114+
const isMatch =
112115
skillLangs.some((l) => projectLanguages.has(l)) ||
113-
skillFw.some((f) => projectFrameworks.has(f))
114-
);
115-
});
116+
skillFw.some((f) => projectFrameworks.has(f));
117+
(isMatch ? matching : other).push(skill);
118+
}
119+
120+
// Combine: matching skills first, then others, capped at 30 total
121+
const MAX_SKILLS = 30;
122+
const combined = [...matching, ...other].slice(0, MAX_SKILLS);
116123

117-
if (relevant.length > 0) {
124+
const formatSkill = (skill: GitHubCommunitySkill): string => {
125+
const tags = skill.tags?.join(', ') ?? '';
126+
const languages = skill.languages?.join(', ') ?? '';
127+
const frameworks = skill.frameworks?.join(', ') ?? '';
128+
return `- **${skill.id}**: ${skill.description} [category: ${skill.category}] [tags: ${tags}] [languages: ${languages}] [frameworks: ${frameworks}]`;
129+
};
130+
131+
if (combined.length > 0) {
118132
parts.push('');
119-
parts.push(`## Matching Skills (${relevant.length} match project stack)`);
120-
for (const skill of relevant.slice(0, 15)) {
121-
const tags = skill.tags?.join(', ') ?? '';
122-
const languages = skill.languages?.join(', ') ?? '';
123-
const frameworks = skill.frameworks?.join(', ') ?? '';
124-
parts.push(
125-
`- **${skill.id}**: ${skill.description} [category: ${skill.category}] [tags: ${tags}] [languages: ${languages}] [frameworks: ${frameworks}]`,
126-
);
133+
if (matching.length > 0) {
134+
parts.push(`## Matching Skills (${matching.length} match project stack)`);
135+
for (const skill of matching.slice(0, MAX_SKILLS)) {
136+
parts.push(formatSkill(skill));
137+
}
138+
}
139+
const otherToShow = combined.length - matching.length;
140+
if (otherToShow > 0) {
141+
parts.push('');
142+
parts.push('## Other Skills');
143+
for (const skill of other.slice(0, otherToShow)) {
144+
parts.push(formatSkill(skill));
145+
}
127146
}
128-
if (relevant.length > 15) {
129-
parts.push(` ... and ${relevant.length - 15} more matching skills`);
147+
if (registrySkills.length > MAX_SKILLS) {
148+
parts.push(` ... and ${registrySkills.length - MAX_SKILLS} more skills`);
130149
}
131150
}
132151

tests/skills/learnPrompts.test.ts

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,10 @@ describe('learnPrompts', () => {
199199
});
200200

201201
describe('buildLearnUserPrompt registry handling', () => {
202-
it('does not dump full registry when skills exceed threshold', () => {
202+
it('caps skills at 30 and mentions overflow', () => {
203203
const analysis = makeAnalysis({ languages: ['typescript'], frameworks: ['react'] });
204204

205-
// Generate a large registry with no language/framework match
205+
// Generate a large registry
206206
const manySkills = Array.from({ length: 50 }, (_, i) =>
207207
makeRegistrySkill({
208208
id: `skill-${i}`,
@@ -215,22 +215,29 @@ describe('learnPrompts', () => {
215215

216216
const prompt = buildLearnUserPrompt(analysis, [], manySkills);
217217

218-
// Should mention find_agent_skills, not list all 50
218+
// Should cap at 30 and mention overflow
219219
expect(prompt).toContain('find_agent_skills');
220-
expect(prompt).not.toContain('skill-49');
220+
expect(prompt).toContain('skill-0');
221+
expect(prompt).toContain('skill-29');
222+
expect(prompt).not.toContain('skill-30');
221223
});
222224

223-
it('shows matching skills filtered by project stack', () => {
225+
it('lists matching skills first, then others', () => {
224226
const analysis = makeAnalysis({ languages: ['typescript'], frameworks: ['react'] });
225227

226228
const skills = [
227-
makeRegistrySkill({ id: 'ts-skill', languages: ['typescript'], frameworks: [] }),
228229
makeRegistrySkill({ id: 'ruby-skill', languages: ['ruby'], frameworks: ['rails'] }),
230+
makeRegistrySkill({ id: 'ts-skill', languages: ['typescript'], frameworks: [] }),
229231
];
230232

231233
const prompt = buildLearnUserPrompt(analysis, [], skills);
234+
// Both should be visible to the LLM
232235
expect(prompt).toContain('ts-skill');
233-
expect(prompt).not.toContain('ruby-skill');
236+
expect(prompt).toContain('ruby-skill');
237+
// Matching skill should appear before non-matching (in "Matching Skills" section)
238+
const tsIdx = prompt.indexOf('ts-skill');
239+
const rubyIdx = prompt.indexOf('ruby-skill');
240+
expect(tsIdx).toBeLessThan(rubyIdx);
234241
});
235242

236243
it('includes registry count summary', () => {
@@ -248,6 +255,36 @@ describe('learnPrompts', () => {
248255
const prompt = buildLearnUserPrompt(analysis, [], skills);
249256
expect(prompt).toContain('find_agent_skills');
250257
});
258+
259+
it('includes skills with empty languages/frameworks (no metadata)', () => {
260+
const analysis = makeAnalysis({ languages: ['typescript'], frameworks: ['react'] });
261+
262+
const skills = [
263+
makeRegistrySkill({ id: 'error-handling', description: 'Error patterns', languages: [], frameworks: [], tags: ['patterns'] }),
264+
makeRegistrySkill({ id: 'ts-skill', languages: ['typescript'], frameworks: [] }),
265+
];
266+
267+
const prompt = buildLearnUserPrompt(analysis, [], skills);
268+
// Skills with no metadata should still appear — the LLM should decide relevance
269+
expect(prompt).toContain('error-handling');
270+
expect(prompt).toContain('ts-skill');
271+
});
272+
273+
it('includes all skills when total count is within the limit', () => {
274+
const analysis = makeAnalysis({ languages: ['typescript'], frameworks: ['react'] });
275+
276+
const skills = [
277+
makeRegistrySkill({ id: 'ts-skill', languages: ['typescript'], frameworks: [] }),
278+
makeRegistrySkill({ id: 'go-skill', languages: ['go'], frameworks: [] }),
279+
makeRegistrySkill({ id: 'generic-skill', languages: [], frameworks: [] }),
280+
];
281+
282+
const prompt = buildLearnUserPrompt(analysis, [], skills);
283+
// All 3 skills should be visible to the LLM for ranking
284+
expect(prompt).toContain('ts-skill');
285+
expect(prompt).toContain('go-skill');
286+
expect(prompt).toContain('generic-skill');
287+
});
251288
});
252289

253290
describe('buildLearnGenerationSystemPrompt', () => {

0 commit comments

Comments
 (0)