chore: apply pr-reviewer fixes for #1092

Author: agent-relay-code[bot]

.trajectories/completed/2026-06/traj_o61z0ze6kvla/summary.md

@@ -0,0 +1,31 @@
+# Trajectory: Review and fix PR #1092
+
+> **Status:** ✅ Completed
+> **Confidence:** 70%
+> **Started:** June 11, 2026 at 08:29 AM
+> **Completed:** June 11, 2026 at 08:30 AM
+
+---
+
+## Summary
+
+Fixed eval parser, runner, summary, executor, and relay-check issues for PR #1092; scoped verification passed, full verification blocked by incomplete dependency install and GitHub mergeability is dirty.
+
+**Approach:** Standard approach
+
+---
+
+## Key Decisions
+
+### Kept fixes scoped to relay eval harness
+- **Chose:** Kept fixes scoped to relay eval harness
+- **Reasoning:** Validated current PR comments and changed only PR eval scripts/checks; full repo verification is blocked by killed npm ci leaving missing dependencies.
+
+---
+
+## Chapters
+
+### 1. Work
+*Agent: default*
+
+- Kept fixes scoped to relay eval harness: Kept fixes scoped to relay eval harness

.trajectories/completed/2026-06/traj_o61z0ze6kvla/trajectory.json

@@ -0,0 +1,53 @@
+{
+  "id": "traj_o61z0ze6kvla",
+  "version": 1,
+  "task": {
+    "title": "Review and fix PR #1092"
+  },
+  "status": "completed",
+  "startedAt": "2026-06-11T08:29:58.788Z",
+  "completedAt": "2026-06-11T08:30:00.119Z",
+  "agents": [
+    {
+      "name": "default",
+      "role": "lead",
+      "joinedAt": "2026-06-11T08:29:59.481Z"
+    }
+  ],
+  "chapters": [
+    {
+      "id": "chap_f292s4revkwp",
+      "title": "Work",
+      "agentName": "default",
+      "startedAt": "2026-06-11T08:29:59.481Z",
+      "endedAt": "2026-06-11T08:30:00.119Z",
+      "events": [
+        {
+          "ts": 1781166599482,
+          "type": "decision",
+          "content": "Kept fixes scoped to relay eval harness: Kept fixes scoped to relay eval harness",
+          "raw": {
+            "question": "Kept fixes scoped to relay eval harness",
+            "chosen": "Kept fixes scoped to relay eval harness",
+            "alternatives": [],
+            "reasoning": "Validated current PR comments and changed only PR eval scripts/checks; full repo verification is blocked by killed npm ci leaving missing dependencies."
+          },
+          "significance": "high"
+        }
+      ]
+    }
+  ],
+  "retrospective": {
+    "summary": "Fixed eval parser, runner, summary, executor, and relay-check issues for PR #1092; scoped verification passed, full verification blocked by incomplete dependency install and GitHub mergeability is dirty.",
+    "approach": "Standard approach",
+    "confidence": 0.7
+  },
+  "commits": [],
+  "filesChanged": [],
+  "projectId": "/home/daytona/workspace",
+  "tags": [],
+  "_trace": {
+    "startRef": "5e63ef398e6376b6c96d9cffeae9c1b668ab45cf",
+    "endRef": "5e63ef398e6376b6c96d9cffeae9c1b668ab45cf"
+  }
+}

scripts/evals/ci-summary.mjs

@@ -1,52 +1,54 @@
 #!/usr/bin/env node
 
-import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 
-const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
-const RUNS_DIR = path.join(ROOT, ".relay", "evals", "runs");
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+const RUNS_DIR = path.join(ROOT, '.relay', 'evals', 'runs');
 
 const runDir = findLatestRunDir();
 if (!runDir) {
-  const summary = "# Relay Eval CI Summary\n\nNo Relay eval run found.\n";
+  const summary = '# Relay Eval CI Summary\n\nNo Relay eval run found.\n';
   console.log(summary);
-  if (process.env.GITHUB_STEP_SUMMARY) writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: "a" });
+  if (process.env.GITHUB_STEP_SUMMARY) writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' });
   process.exit(0);
 }
 
-const resultPath = path.join(runDir, "result.json");
+const resultPath = path.join(runDir, 'result.json');
 const result = readResultJson(resultPath);
-const failed = result.tests.filter((test) => test.status === "failed");
-const skipped = result.tests.filter((test) => test.status === "skipped");
-const needsHuman = result.tests.filter((test) => test.status === "needs-human");
+const failed = result.tests.filter((test) => test.status === 'failed');
+const skipped = result.tests.filter((test) => test.status === 'skipped');
+const needsHuman = result.tests.filter((test) => test.status === 'needs-human');
 
 const lines = [
-  "# Relay Eval CI Summary",
-  "",
+  '# Relay Eval CI Summary',
+  '',
   `- Run directory: \`${path.relative(ROOT, runDir)}\``,
   `- Mode: \`${result.mode}\``,
   `- Git SHA: \`${result.git_sha}\``,
   `- Passed: ${result.passed}`,
   `- Needs human review: ${result.needs_human}`,
   `- Failed: ${result.failed}`,
   `- Skipped: ${result.skipped}`,
-  "",
+  '',
 ];
 
-appendStatusSection(lines, "Failed", failed);
-appendStatusSection(lines, "Skipped", skipped);
+appendStatusSection(lines, 'Failed', failed);
+appendStatusSection(lines, 'Skipped', skipped);
 appendNeedsHumanSection(lines, needsHuman);
 
-const summary = `${lines.join("\n")}\n`;
+const summary = `${lines.join('\n')}\n`;
 console.log(summary);
 
-if (process.env.GITHUB_STEP_SUMMARY) writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: "a" });
-if (failed.length > 0 || skipped.length > 0) process.exitCode = 1;
+if (process.env.GITHUB_STEP_SUMMARY) writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' });
+const failOnSkipped =
+  process.env.RELAY_EVAL_FAIL_ON_SKIPPED === '1' || process.env.HUMAN_EVAL_FAIL_ON_SKIPPED === '1';
+if (failed.length > 0 || (failOnSkipped && skipped.length > 0)) process.exitCode = 1;
 
 function appendStatusSection(lines, title, tests) {
   if (tests.length === 0) return;
-  lines.push(`## ${title}`, "");
+  lines.push(`## ${title}`, '');
   for (const test of tests) {
     lines.push(`- \`${test.id}\` (${test.suite}/${test.executor})`);
     if (test.error) lines.push(`  - ${test.error}`);
@@ -55,26 +57,26 @@ function appendStatusSection(lines, title, tests) {
       lines.push(`  - FAIL ${check.name}: ${check.message}`);
     }
   }
-  lines.push("");
+  lines.push('');
 }
 
 function appendNeedsHumanSection(lines, tests) {
-  lines.push("## Human Review", "");
+  lines.push('## Human Review', '');
   if (tests.length === 0) {
-    lines.push("No cases require human review.", "");
+    lines.push('No cases require human review.', '');
     return;
   }
   for (const test of tests) lines.push(`- \`${test.id}\` (${test.suite}/${test.executor})`);
-  lines.push("");
+  lines.push('');
 }
 
 function findLatestRunDir() {
   if (!existsSync(RUNS_DIR)) return null;
   const runs = readdirSync(RUNS_DIR)
     .map((dir) => path.join(RUNS_DIR, dir))
-    .filter((dir) => existsSync(path.join(dir, "result.json")))
+    .filter((dir) => existsSync(path.join(dir, 'result.json')))
     .flatMap((dir) => {
-      const result = safeReadResultJson(path.join(dir, "result.json"));
+      const result = safeReadResultJson(path.join(dir, 'result.json'));
       return result ? [{ dir, result }] : [];
     })
     .sort((a, b) => String(b.result.timestamp).localeCompare(String(a.result.timestamp)));
@@ -89,9 +91,11 @@ function readResultJson(filePath) {
 
 function safeReadResultJson(filePath) {
   try {
-    return JSON.parse(readFileSync(filePath, "utf8"));
+    return JSON.parse(readFileSync(filePath, 'utf8'));
   } catch (error) {
-    console.warn(`Skipping malformed Relay eval result ${path.relative(ROOT, filePath)}: ${error instanceof Error ? error.message : String(error)}`);
+    console.warn(
+      `Skipping malformed Relay eval result ${path.relative(ROOT, filePath)}: ${error instanceof Error ? error.message : String(error)}`
+    );
     return null;
   }
 }