Skip to content

Commit c7c642f

Browse files
CopilotCopilot
authored
fix(log_parser_bootstrap): guard Claude guardrail with safeOutputEntriesCount escape hatch (#41886)
1 parent c36a756 commit c7c642f

2 files changed

Lines changed: 113 additions & 3 deletions

File tree

‎actions/setup/js/log_parser_bootstrap.cjs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,13 @@ async function runLogParser(options) {
297297

298298
core.summary.addRaw(fullMarkdown).write();
299299
} else {
300-
// Fallback: just log success message for parsers without log entries
301-
core.info(`${parserName} log parsed successfully`);
300+
// Fallback path: markdown exists but no structured log entries were parsed.
301+
// Suppress the "parsed successfully" message for Claude since it always produces
302+
// logEntries when healthy — absence of entries means the parse fell back and is
303+
// about to emit a guardrail warning/failure below.
304+
if (parserName !== "Claude") {
305+
core.info(`${parserName} log parsed successfully`);
306+
}
302307

303308
// Add safe outputs preview to core.info (fallback path)
304309
if (safeOutputsContent) {
@@ -330,8 +335,17 @@ async function runLogParser(options) {
330335

331336
// Claude-specific guardrail: if no structured log entries were parsed, treat as execution failure.
332337
// This catches silent startup failures where Claude exits before producing JSON tool activity.
338+
// Exception: when safeOutputEntriesCount > 0 the agent demonstrably completed and emitted
339+
// safe outputs — treat as a non-fatal post-completion infrastructure failure (e.g. sandbox
340+
// teardown race leaving agent-stdio.log unreadable) and downgrade to a warning.
333341
if (parserName === "Claude" && (!logEntries || logEntries.length === 0)) {
334-
core.setFailed(`${ERR_CONFIG}: Claude execution failed: no structured log entries were produced. This usually indicates a startup or configuration error before tool execution.`);
342+
if (safeOutputEntriesCount > 0) {
343+
core.warning(
344+
`Claude produced no structured log entries, but agent completed with ${safeOutputEntriesCount} safe output ${safeOutputEntriesCount === 1 ? "entry" : "entries"} — treating as non-fatal post-completion infrastructure failure`
345+
);
346+
} else {
347+
core.setFailed(`${ERR_CONFIG}: Claude execution failed: no structured log entries were produced. This usually indicates a startup or configuration error before tool execution.`);
348+
}
335349
}
336350

337351
// Handle MCP server failures if present

‎actions/setup/js/log_parser_bootstrap.test.cjs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,102 @@ describe("log_parser_bootstrap.cjs", () => {
203203
fs.rmdirSync(tmpDir);
204204
}
205205
}),
206+
it("should warn (non-fatal) when Claude has empty logEntries but safe outputs exist", () => {
207+
const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-"));
208+
const logFile = path.join(tmpDir, "test.log");
209+
const safeOutputsFile = path.join(tmpDir, "safe-outputs.jsonl");
210+
try {
211+
fs.writeFileSync(logFile, "some raw content");
212+
fs.writeFileSync(safeOutputsFile, JSON.stringify({ type: "create_issue", title: "Test", body: "Test body" }));
213+
process.env.GH_AW_AGENT_OUTPUT = logFile;
214+
process.env.GH_AW_SAFE_OUTPUTS = safeOutputsFile;
215+
// Parser returns markdown but no structured logEntries — simulates sandbox teardown
216+
// race leaving agent-stdio.log unreadable after the agent completed successfully.
217+
const mockParseLog = vi.fn().mockReturnValue({ markdown: "## Result\n", mcpFailures: [], maxTurnsHit: false, logEntries: [] });
218+
runLogParser({ parseLog: mockParseLog, parserName: "Claude" });
219+
expect(mockCore.setFailed).not.toHaveBeenCalled();
220+
expect(mockCore.warning).toHaveBeenCalledWith("Claude produced no structured log entries, but agent completed with 1 safe output entry — treating as non-fatal post-completion infrastructure failure");
221+
} finally {
222+
fs.unlinkSync(logFile);
223+
fs.unlinkSync(safeOutputsFile);
224+
delete process.env.GH_AW_SAFE_OUTPUTS;
225+
fs.rmdirSync(tmpDir);
226+
}
227+
}),
228+
it("should fail when Claude has empty logEntries and no safe outputs (startup failure)", () => {
229+
const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-"));
230+
const logFile = path.join(tmpDir, "test.log");
231+
try {
232+
fs.writeFileSync(logFile, "some raw content");
233+
process.env.GH_AW_AGENT_OUTPUT = logFile;
234+
delete process.env.GH_AW_SAFE_OUTPUTS;
235+
const mockParseLog = vi.fn().mockReturnValue({ markdown: "## Result\n", mcpFailures: [], maxTurnsHit: false, logEntries: [] });
236+
runLogParser({ parseLog: mockParseLog, parserName: "Claude" });
237+
expect(mockCore.setFailed).toHaveBeenCalledWith(`${ERR_CONFIG}: Claude execution failed: no structured log entries were produced. This usually indicates a startup or configuration error before tool execution.`);
238+
} finally {
239+
fs.unlinkSync(logFile);
240+
fs.rmdirSync(tmpDir);
241+
}
242+
}),
243+
it("should not print 'parsed successfully' for Claude when logEntries is empty", () => {
244+
const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-"));
245+
const logFile = path.join(tmpDir, "test.log");
246+
const safeOutputsFile = path.join(tmpDir, "safe-outputs.jsonl");
247+
try {
248+
fs.writeFileSync(logFile, "some raw content");
249+
fs.writeFileSync(safeOutputsFile, JSON.stringify({ type: "add_comment", body: "Done" }));
250+
process.env.GH_AW_AGENT_OUTPUT = logFile;
251+
process.env.GH_AW_SAFE_OUTPUTS = safeOutputsFile;
252+
const mockParseLog = vi.fn().mockReturnValue({ markdown: "## Result\n", mcpFailures: [], maxTurnsHit: false, logEntries: [] });
253+
runLogParser({ parseLog: mockParseLog, parserName: "Claude" });
254+
const infoCalls = mockCore.info.mock.calls.map(c => c[0]);
255+
expect(infoCalls.some(msg => msg.includes("Claude log parsed successfully"))).toBe(false);
256+
expect(mockCore.setFailed).not.toHaveBeenCalled();
257+
expect(mockCore.warning).toHaveBeenCalledWith("Claude produced no structured log entries, but agent completed with 1 safe output entry — treating as non-fatal post-completion infrastructure failure");
258+
} finally {
259+
fs.unlinkSync(logFile);
260+
fs.unlinkSync(safeOutputsFile);
261+
delete process.env.GH_AW_SAFE_OUTPUTS;
262+
fs.rmdirSync(tmpDir);
263+
}
264+
}),
265+
it("should treat logEntries: null as missing entries for Claude guardrail (no safe outputs → setFailed)", () => {
266+
const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-"));
267+
const logFile = path.join(tmpDir, "test.log");
268+
try {
269+
fs.writeFileSync(logFile, "some raw content");
270+
process.env.GH_AW_AGENT_OUTPUT = logFile;
271+
delete process.env.GH_AW_SAFE_OUTPUTS;
272+
const mockParseLog = vi.fn().mockReturnValue({ markdown: "## Result\n", mcpFailures: [], maxTurnsHit: false, logEntries: null });
273+
runLogParser({ parseLog: mockParseLog, parserName: "Claude" });
274+
expect(mockCore.setFailed).toHaveBeenCalledWith(`${ERR_CONFIG}: Claude execution failed: no structured log entries were produced. This usually indicates a startup or configuration error before tool execution.`);
275+
} finally {
276+
fs.unlinkSync(logFile);
277+
delete process.env.GH_AW_AGENT_OUTPUT;
278+
fs.rmdirSync(tmpDir);
279+
}
280+
}),
281+
it("should treat logEntries: null as missing entries for Claude guardrail (safe outputs → warning)", () => {
282+
const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-"));
283+
const logFile = path.join(tmpDir, "test.log");
284+
const safeOutputsFile = path.join(tmpDir, "safe-outputs.jsonl");
285+
try {
286+
fs.writeFileSync(logFile, "some raw content");
287+
fs.writeFileSync(safeOutputsFile, JSON.stringify({ type: "create_issue", title: "Test", body: "Test body" }));
288+
process.env.GH_AW_AGENT_OUTPUT = logFile;
289+
process.env.GH_AW_SAFE_OUTPUTS = safeOutputsFile;
290+
const mockParseLog = vi.fn().mockReturnValue({ markdown: "## Result\n", mcpFailures: [], maxTurnsHit: false, logEntries: null });
291+
runLogParser({ parseLog: mockParseLog, parserName: "Claude" });
292+
expect(mockCore.setFailed).not.toHaveBeenCalled();
293+
expect(mockCore.warning).toHaveBeenCalledWith("Claude produced no structured log entries, but agent completed with 1 safe output entry — treating as non-fatal post-completion infrastructure failure");
294+
} finally {
295+
fs.unlinkSync(logFile);
296+
fs.unlinkSync(safeOutputsFile);
297+
delete process.env.GH_AW_AGENT_OUTPUT;
298+
delete process.env.GH_AW_SAFE_OUTPUTS;
299+
fs.rmdirSync(tmpDir);
300+
}
301+
}),
206302
it("should handle max-turns limit reached", () => {
207303
const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-")),
208304
logFile = path.join(tmpDir, "test.log");

0 commit comments

Comments
 (0)