fix: surface hosted review runtime failures (#11)

Author: CompleteDotTech

src/adapter.ts

@@ -51,7 +51,7 @@ export interface AdapterResponse {
   body: JsonObject;
 }
 
-interface CommandResult extends JsonObject {
+export interface CommandResult extends JsonObject {
   args: string[];
   returncode: number;
   stdout: string;
@@ -179,7 +179,7 @@ function createFreshChildDirectory(parent: string, name: string, label: string):
   return path;
 }
 
-function createFreshTaskAttemptDirectory(root: string, taskId: string, attempt: number, label: string): string {
+export function createFreshTaskAttemptDirectory(root: string, taskId: string, attempt: number, label: string): string {
   if (!validRecordId(taskId)) throw new Error(`Invalid task ID for ${label}`);
   if (!Number.isSafeInteger(attempt) || attempt <= 0) throw new Error(`Invalid attempt number for ${label}`);
   const taskRoot = ensureManagedChildDirectory(root, taskId, `${label} task directory`);
@@ -1613,6 +1613,7 @@ async function runTaskUnlocked(config: AdapterConfig, taskId: string): Promise<J
     task.session_brief_path = firstCycle.brief_path;
     task.session_brief_sha256 = fileSha256(String(firstCycle.brief_path));
     task.runtime_exit_code = firstCycle.run.returncode;
+    task.runtime_diagnostic = runtimeDiagnostic(firstCycle.run) || undefined;
     task.result_path = firstCycle.result_path;
     writeJsonAtomic(path, task);
 
@@ -1641,6 +1642,7 @@ async function runTaskUnlocked(config: AdapterConfig, taskId: string): Promise<J
       });
       task.review_fix_loops = loopRecords;
       task.runtime_exit_code = repairCycle.run.returncode;
+      task.runtime_diagnostic = runtimeDiagnostic(repairCycle.run) || undefined;
       task.result_path = repairCycle.result_path;
       task.updated_at = utcNow();
       writeJsonAtomic(path, task);
@@ -3588,6 +3590,10 @@ function publicationCommentBody(task: JsonObject, result: JsonObject, heading =
   if (prBody.trim() && prBody.trim() !== summary.trim()) {
     parts.push("", prBody.trim());
   }
+  const diagnostic = safePublicationText(String(task.runtime_diagnostic || ""), 1000).trim();
+  if (diagnostic && result.status !== "success") {
+    parts.push("", "### Runtime diagnostic", diagnostic);
+  }
   parts.push(...reviewFixLoopLines(task), "", "### Evidence");
   if (Object.keys(evidence).length) {
     const changedFiles = Array.isArray(evidence.changed_files) ? evidence.changed_files : [];
@@ -3741,13 +3747,37 @@ function codexTokenCandidates(config: AdapterConfig): string[] {
 }
 
 function redactedCommandResult(result: CommandResult): JsonObject {
+  const diagnostic = runtimeDiagnostic(result);
   return {
     ...result,
     stdout: redactTokenish(result.stdout),
     stderr: redactTokenish(result.stderr),
+    ...(diagnostic ? {runtime_diagnostic: diagnostic} : {}),
   };
 }
 
+export function runtimeDiagnostic(result: CommandResult): string | null {
+  if (result.returncode === 0 && !result.signal && !result.timed_out && !result.spawn_error) {
+    return null;
+  }
+  if (result.timed_out) {
+    return "Coven Code timed out before completing the hosted review.";
+  }
+  if (result.signal) {
+    return `Coven Code stopped after signal ${safePublicationText(String(result.signal), 40)}.`;
+  }
+  const raw = String(result.stderr || result.spawn_error || "").trim();
+  if (!raw) {
+    return `Coven Code exited ${result.returncode} without a diagnostic.`;
+  }
+  const safe = redactTokenish(raw)
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[redacted email]")
+    .replace(/\bon account\s+[^\n.]+/gi, "on the configured account")
+    .replace(/\s+/g, " ")
+    .trim();
+  return safePublicationText(safe, 1000);
+}
+
 export function sanitizedRuntimeEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
   const env: NodeJS.ProcessEnv = {};
   for (const key of [
@@ -3782,6 +3812,8 @@ export function redactTokenish(text: string): string {
     .replace(/\b(?:gh[pousr]_|github_pat_)[A-Za-z0-9_-]{6,}/g, "[redacted github token]")
     .replace(/\bsk-(?:proj-)?[A-Za-z0-9_-]{8,}/g, "[redacted OpenAI token]")
     .replace(/\bBearer\s+[^\s'\"]+/gi, "Bearer [redacted]")
+    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[redacted email]")
+    .replace(/\bon account\s+`?[^`\n.]+`?/gi, "on the configured account")
     .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[redacted JWT]")
     .replace(/x-access-token:[^@\s'\"]+/gi, "x-access-token:[redacted]")
     .replace(/(https?:\/\/)[^/\s:@]+:[^@\s/]+@/gi, "$1[redacted]@");

tests/webhook-adapter.test.ts

@@ -10,6 +10,7 @@ import { pathToFileURL } from "node:url";
 
 import {
   buildTaskFromEvent,
+  createFreshTaskAttemptDirectory,
   createConfig,
   githubRequestAllPages,
   handleRequest,
@@ -24,6 +25,7 @@ import {
   resumeTaskPublication,
   runtimeInstallationTokenRequest,
   runtimeIsolationIssue,
+  runtimeDiagnostic,
   runtimeProcessEnvironment,
   runtimeSandboxArgs,
   runnableTaskIds,
@@ -2893,11 +2895,12 @@ test("redacts credentials and passes only allowlisted ambient environment keys",
   const secretText = [
     "ghs_1234567890", "ghp_1234567890", "github_pat_1234567890",
     "sk-proj-1234567890", "Bearer topsecret", "eyJabc.def.ghi",
+    "API error 429 on account `reviewer (reviewer@example.com)`.",
     "https://user:password@example.com/path",
     "-----BEGIN PRIVATE KEY-----\nprivate-data\n-----END PRIVATE KEY-----",
   ].join("\n");
   const redacted = redactTokenish(secretText);
-  assert.doesNotMatch(redacted, /1234567890|topsecret|password|private-data|eyJabc/);
+  assert.doesNotMatch(redacted, /1234567890|topsecret|password|private-data|eyJabc|reviewer@example\.com|reviewer \(/);
   const env = sanitizedRuntimeEnvironment({
     PATH: "/bin", LANG: "C.UTF-8", SSH_AUTH_SOCK: "/tmp/agent.sock",
     DATABASE_URL: "postgres://user:pass@db", AWS_ACCESS_KEY_ID: "AKIASECRET",
@@ -2906,6 +2909,57 @@ test("redacts credentials and passes only allowlisted ambient environment keys",
   assert.deepEqual(env, {PATH: "/bin", LANG: "C.UTF-8"});
 });
 
+test("preserves a useful redacted provider diagnostic for failed reviews", async () => {
+  const diagnostic = runtimeDiagnostic({
+    args: ["coven-code", "--headless"],
+    returncode: 2,
+    stdout: "",
+    stderr: "API error 429: rate limit for model `gpt-5.4-mini` on account `reviewer (reviewer@example.com)`. Bearer topsecret sk-proj-1234567890\nProvider: codex",
+    signal: null,
+    timed_out: false,
+    spawn_error: "",
+  });
+  assert.match(String(diagnostic), /API error 429/);
+  assert.match(String(diagnostic), /gpt-5\.4-mini/);
+  assert.match(String(diagnostic), /configured account/);
+  assert.doesNotMatch(String(diagnostic), /reviewer@example\.com|reviewer \(|topsecret|1234567890/);
+
+  const stateDir = tempStateDir();
+  const config = testConfig(stateDir);
+  const task = reviewTask("provider-failure-diagnostic");
+  task.runtime_diagnostic = diagnostic;
+  prepareReviewWorkspace(config, task);
+  const result = completeReview();
+  result.status = "failure";
+  const resultPath = join(stateDir, "provider-failure.json");
+  writeFileSync(resultPath, JSON.stringify(result));
+  let payload: JsonObject = {};
+  await withGithubApiMock((url, init) => {
+    const read = githubReadFixture(url, init);
+    if (read) return read;
+    payload = JSON.parse(String(init.body)) as JsonObject;
+    return {id: 409, state: "PENDING", html_url: "https://github.com/OpenCoven/example/pull/7#pullrequestreview-409"};
+  }, async () => publishResultIfConfigured(config, task, resultPath, "token"));
+  assert.equal(payload.event, "COMMENT");
+  assert.match(String(payload.body), /### Runtime diagnostic/);
+  assert.match(String(payload.body), /API error 429/);
+  assert.doesNotMatch(String(payload.body), /reviewer@example\.com|reviewer \(|topsecret|1234567890/);
+});
+
+test("creates isolated attempt directories and refuses stale artifact reuse", () => {
+  const root = tempStateDir();
+  const first = createFreshTaskAttemptDirectory(root, "review-task", 1, "test attempt");
+  writeFileSync(join(first, "run.json"), "stale artifact\n");
+  const second = createFreshTaskAttemptDirectory(root, "review-task", 2, "test attempt");
+  assert.notEqual(first, second);
+  assert.equal(readFileSync(join(first, "run.json"), "utf8"), "stale artifact\n");
+  assert.deepEqual(readdirSync(second), []);
+  assert.throws(
+    () => createFreshTaskAttemptDirectory(root, "review-task", 1, "test attempt"),
+    /already exists/,
+  );
+});
+
 test("keeps idempotency marker after truncating and redacts issue publication text", async () => {
   const stateDir = tempStateDir();
   const task: JsonObject = {