Skip to content

Commit 359d482

Browse files
bronz3beardbronz3beard
committed
refactor(reflexion): improve YAML block extraction and validation in runner
1 parent 78779e4 commit 359d482

3 files changed

Lines changed: 222 additions & 47 deletions

File tree

‎scripts/__tests__/reflexion-issue-runner-utils.test.js‎

Lines changed: 63 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,69 +1,115 @@
1-
const { extractRunIdMarker, extractProcessedCommentIdMarker, formatRunnerComment, extractYamlBlock, validateAnswers } = require('../reflexion-issue-runner-utils.js');
1+
const {
2+
extractRunIdMarker,
3+
extractProcessedCommentIdMarker,
4+
formatRunnerComment,
5+
extractYamlBlock,
6+
validateAnswers,
7+
} = require('../reflexion-issue-runner-utils.js');
28

39
describe('reflexion-issue-runner-utils', () => {
410
describe('extractRunIdMarker', () => {
511
it('should extract a valid marker', () => {
6-
const body = "Some text\n<!-- reflexion-run:987654321 -->\nMore text";
7-
expect(extractRunIdMarker(body)).toBe("987654321");
12+
const body = 'Some text\n<!-- reflexion-run:987654321 -->\nMore text';
13+
expect(extractRunIdMarker(body)).toBe('987654321');
814
});
915
});
1016

1117
describe('extractProcessedCommentIdMarker', () => {
1218
it('should extract a valid marker', () => {
13-
const body = "Some text\n<!-- processed-comment-id:444555 -->\nMore text";
14-
expect(extractProcessedCommentIdMarker(body)).toBe("444555");
19+
const body = 'Some text\n<!-- processed-comment-id:444555 -->\nMore text';
20+
expect(extractProcessedCommentIdMarker(body)).toBe('444555');
1521
});
1622
});
1723

1824
describe('formatRunnerComment', () => {
1925
it('should format a diagnostic error message', () => {
2026
const result = formatRunnerComment({
21-
runId: "123",
22-
diagnostic: "Something went wrong"
27+
runId: '123',
28+
diagnostic: 'Something went wrong',
2329
});
2430
expect(result).toContain('<!-- reflexion-run:123 -->');
2531
expect(result).toContain('### 🚨 Reflexion Loop Error');
2632
});
2733

2834
it('should include processed-comment-id marker', () => {
2935
const result = formatRunnerComment({
30-
runId: "123",
31-
triggeringCommentId: "888",
32-
diagnostic: "Something went wrong"
36+
runId: '123',
37+
triggeringCommentId: '888',
38+
diagnostic: 'Something went wrong',
3339
});
3440
expect(result).toContain('<!-- processed-comment-id:888 -->');
3541
});
3642
});
3743

3844
describe('extractYamlBlock', () => {
3945
it('should extract a standard yaml block', () => {
40-
const body = "Here is the yaml:\n```yaml\nfoo: bar\n```";
41-
expect(extractYamlBlock(body).trim()).toBe("foo: bar");
46+
const body = 'Here is the yaml:\n```yaml\nfoo: bar\n```';
47+
expect(extractYamlBlock(body).trim()).toBe('foo: bar');
48+
});
49+
50+
it('should extract a yml block', () => {
51+
const body = 'Here is the yml:\n```yml\nfoo: bar\n```';
52+
expect(extractYamlBlock(body).trim()).toBe('foo: bar');
53+
});
54+
55+
it('should extract a yaml answers: block', () => {
56+
const body = 'Here is the answers:\n```yaml answers:\nfoo: bar\n```';
57+
expect(extractYamlBlock(body).trim()).toBe('foo: bar');
58+
});
59+
60+
it('should extract a yml answers: block', () => {
61+
const body = 'Here is the answers:\n```yml answers:\nfoo: bar\n```';
62+
expect(extractYamlBlock(body).trim()).toBe('foo: bar');
63+
});
64+
65+
it('should extract an answers: block', () => {
66+
const body = 'Here is the answers:\n```answers:\nfoo: bar\n```';
67+
expect(extractYamlBlock(body).trim()).toBe('foo: bar');
68+
});
69+
70+
it('should extract a bare block starting with runId:', () => {
71+
const body = 'Here is the bare block:\n```\nrunId: "123"\n```';
72+
expect(extractYamlBlock(body).trim()).toBe('runId: "123"');
73+
});
74+
75+
it('should extract a bare block starting with answers:', () => {
76+
const body = 'Here is the bare block:\n```\nanswers:\n - id: q1\n```';
77+
expect(extractYamlBlock(body).trim()).toBe('answers:\n - id: q1');
78+
});
79+
80+
it('should return null for non-matching code blocks (e.g., js)', () => {
81+
const body = 'Some js code:\n```js\nconsole.log(1);\n```';
82+
expect(extractYamlBlock(body)).toBeNull();
83+
});
84+
85+
it('should return null for a bare block that does not start with runId or answers', () => {
86+
const body = 'Some text:\n```\nfoo: bar\n```';
87+
expect(extractYamlBlock(body)).toBeNull();
4288
});
4389
});
4490

4591
describe('validateAnswers', () => {
4692
it('validates correct answers structure', () => {
4793
const valid = {
48-
runId: "987",
49-
decisions: [{ id: "q1", answer: "because" }]
94+
runId: '987',
95+
decisions: [{ id: 'q1', answer: 'because' }],
5096
};
5197
const res = validateAnswers(valid);
5298
expect(res.success).toBe(true);
5399
});
54100

55101
it('validates directive only', () => {
56102
const valid = {
57-
runId: "987",
58-
directive: "approve"
103+
runId: '987',
104+
directive: 'approve',
59105
};
60106
const res = validateAnswers(valid);
61107
expect(res.success).toBe(true);
62108
});
63109

64110
it('fails on missing runId', () => {
65111
const invalid = {
66-
decisions: [{ id: "q1", answer: "because" }]
112+
decisions: [{ id: 'q1', answer: 'because' }],
67113
};
68114
const res = validateAnswers(invalid);
69115
expect(res.success).toBe(false);

‎scripts/reflexion-issue-runner-utils.js‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,12 +85,37 @@ function formatRunnerComment(opts) {
8585

8686
function extractYamlBlock(text) {
8787
if (!text) return null;
88-
const match = text.match(/```yaml(?: answers:)?\s*([\s\S]*?)```/);
89-
if (match) return match[1];
90-
91-
const match2 = text.match(/```answers:\s*([\s\S]*?)```/);
92-
if (match2) return match2[1];
88+
const regex = /```([^\n]*)\n([\s\S]*?)```/gi;
89+
let match;
90+
while ((match = regex.exec(text)) !== null) {
91+
const infoString = match[1];
92+
const content = match[2];
93+
const cleanInfo = infoString.trim().toLowerCase().replace(/\s+/g, ' ');
94+
95+
if (
96+
cleanInfo === 'yaml' ||
97+
cleanInfo === 'yml' ||
98+
cleanInfo === 'yaml answers:' ||
99+
cleanInfo === 'yml answers:' ||
100+
cleanInfo === 'answers:'
101+
) {
102+
return content;
103+
}
93104

105+
if (cleanInfo === '') {
106+
const lines = content.split('\n');
107+
const firstNonEmptyLine = lines
108+
.map((l) => l.trim())
109+
.find((l) => l.length > 0);
110+
if (
111+
firstNonEmptyLine &&
112+
(firstNonEmptyLine.startsWith('runId:') ||
113+
firstNonEmptyLine.startsWith('answers:'))
114+
) {
115+
return content;
116+
}
117+
}
118+
}
94119
return null;
95120
}
96121

‎scripts/reflexion-issue-runner.js‎

Lines changed: 129 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,36 @@ async function downloadArtifact(runId, artifactName, token) {
196196
console.log(`Artifact extracted to ${OUT_DIR}`);
197197
}
198198

199+
async function verifyArtifactExists(runId, artifactName, token) {
200+
try {
201+
const repo = process.env.GITHUB_REPOSITORY;
202+
const listUrl = `https://api.github.com/repos/${repo}/actions/runs/${runId}/artifacts`;
203+
204+
const res = await fetch(listUrl, {
205+
headers: {
206+
Authorization: `Bearer ${token}`,
207+
Accept: 'application/vnd.github.v3+json',
208+
},
209+
});
210+
211+
if (!res.ok) {
212+
console.error(
213+
`Failed to list artifacts for run ${runId}: ${res.statusText}`
214+
);
215+
return false;
216+
}
217+
218+
const data = await res.json();
219+
if (!data.artifacts) return false;
220+
221+
const artifact = data.artifacts.find((a) => a.name === artifactName);
222+
return !!artifact;
223+
} catch (err) {
224+
console.error(`Error verifying artifact for run ${runId}:`, err);
225+
return false;
226+
}
227+
}
228+
199229
async function handleResumePath(payload) {
200230
const comment = payload.comment;
201231
const issue = payload.issue;
@@ -249,44 +279,118 @@ async function handleResumePath(payload) {
249279
throw new Error('Hard stop: Maximum of 10 loop turns per issue reached.');
250280
}
251281

252-
const yamlBlock = extractYamlBlock(comment.body);
253-
if (!yamlBlock) {
254-
throw new Error('No YAML block found in the comment.');
255-
}
256-
257-
let parsedYaml;
258-
try {
259-
parsedYaml = yaml.load(yamlBlock);
260-
} catch (e) {
261-
throw new Error('Failed to parse YAML block: ' + e.message);
262-
}
263-
264-
const validation = validateAnswers(parsedYaml);
265-
if (!validation.success) {
266-
throw new Error(
267-
'YAML block schema validation failed:\n' + validation.error.message
268-
);
269-
}
270-
271282
let prevRunId = null;
283+
const artifactName = `reflexion-state-${issue.number}`;
284+
272285
for (let i = comments.length - 1; i >= 0; i--) {
273286
const c = comments[i];
274287
if (c.user.login === 'github-actions[bot]') {
275288
const runId = extractRunIdMarker(c.body);
276289
if (runId) {
277-
prevRunId = runId;
278-
break;
290+
console.log(
291+
`Checking if artifact exists for candidate runId: ${runId}`
292+
);
293+
const exists = await verifyArtifactExists(runId, artifactName, TOKEN);
294+
if (exists) {
295+
prevRunId = runId;
296+
break;
297+
} else {
298+
console.log(
299+
`Artifact not found or expired for runId: ${runId}. Skipping.`
300+
);
301+
}
279302
}
280303
}
281304
}
282305

283306
if (!prevRunId) {
284-
throw new Error(
285-
'Could not find a previous run ID marker from a bot comment to resume from.'
286-
);
307+
const msg = `### 🚨 Prior State Not Found\n\nThe prior reflexion state could not be found. The artifact may have expired after the 7-day retention period, or the previous runs failed to produce a valid state.\n\nPlease start a fresh run by closing/reopening the issue, or triggering a new run manually.`;
308+
await createIssueComment(issue.number, msg);
309+
process.exit(1);
287310
}
288311

289-
await downloadArtifact(prevRunId, `reflexion-state-${issue.number}`, TOKEN);
312+
// Download the artifact first, because we want to extract the state/questions
313+
// and we also need it for resume if the comment is valid.
314+
try {
315+
await downloadArtifact(prevRunId, artifactName, TOKEN);
316+
} catch (err) {
317+
const msg = `### 🚨 Failed to Download Prior State\n\nFailed to download prior state artifact for run ID \`${prevRunId}\`: ${err.message}\n\nPlease try starting a fresh run by closing and reopening this issue.`;
318+
await createIssueComment(issue.number, msg);
319+
process.exit(1);
320+
}
321+
322+
// Parse and validate the comment body
323+
let yamlBlock = null;
324+
let parsedYaml = null;
325+
let parseErrorMsg = null;
326+
327+
try {
328+
yamlBlock = extractYamlBlock(comment.body);
329+
if (!yamlBlock) {
330+
throw new Error(
331+
'No YAML block found in your comment. Please ensure your response contains a code block with answers starting with \`\`\`yaml answers:\`\`.'
332+
);
333+
}
334+
335+
try {
336+
parsedYaml = yaml.load(yamlBlock);
337+
} catch (e) {
338+
throw new Error(
339+
`Failed to parse the YAML block. Please ensure it has valid YAML syntax. Details: ${e.message}`
340+
);
341+
}
342+
343+
const validation = validateAnswers(parsedYaml);
344+
if (!validation.success) {
345+
const errors = validation.error.errors
346+
.map((err) => `- \`${err.path.join('.')}\`: ${err.message}`)
347+
.join('\n');
348+
throw new Error(`YAML block schema validation failed:\n${errors}`);
349+
}
350+
} catch (err) {
351+
parseErrorMsg = err.message;
352+
}
353+
354+
if (parseErrorMsg) {
355+
// We had an answer-parsing/validation failure!
356+
// Post a helpful comment with a copy-paste-ready answers template.
357+
let answersTemplate = null;
358+
try {
359+
const stateFiles = fs
360+
.readdirSync(OUT_DIR)
361+
.filter(
362+
(f) =>
363+
f.endsWith('.json') &&
364+
!f.endsWith('-answers.json') &&
365+
!f.endsWith('-critique.json') &&
366+
f !== 'eval.json'
367+
);
368+
if (stateFiles.length > 0) {
369+
const statePath = path.join(OUT_DIR, stateFiles[0]);
370+
const state = JSON.parse(fs.readFileSync(statePath, 'utf-8'));
371+
if (state.interview && state.interview.questions) {
372+
answersTemplate = `\`\`\`yaml answers:\nrunId: "${prevRunId}"\n# directive: "approve"\ndecisions:\n`;
373+
state.interview.questions.forEach((q) => {
374+
answersTemplate += ` - id: "${q.id}"\n answer: ""\n`;
375+
});
376+
answersTemplate += '```';
377+
}
378+
}
379+
} catch (e) {
380+
console.warn(
381+
'Failed to read state file for error template generation:',
382+
e
383+
);
384+
}
385+
386+
if (!answersTemplate) {
387+
answersTemplate = `\`\`\`yaml answers:\nrunId: "${prevRunId}"\n# directive: "approve"\ndecisions:\n - id: "question_id_here"\n answer: ""\n\`\`\``;
388+
}
389+
390+
const msg = `### 🚨 Malformed answers submission\n\nThere was an issue processing your response:\n\n${parseErrorMsg}\n\n#### Please reply again using this copy-paste-ready template:\n\n${answersTemplate}`;
391+
await createIssueComment(issue.number, msg);
392+
process.exit(1);
393+
}
290394

291395
const answersFile = path.join(WORKSPACE, 'reflexion-answers.yaml');
292396
fs.writeFileSync(

0 commit comments

Comments
 (0)