Harden runner baseline validation

Author: egavrin

.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: devagent-runner
+
+      - uses: actions/checkout@v4
+        with:
+          repository: egavrin/devagent-sdk
+          ref: main
+          path: devagent-sdk
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.10"
+
+      - name: Install SDK dependencies
+        working-directory: devagent-sdk
+        run: bun install
+
+      - name: Build SDK packages
+        working-directory: devagent-sdk
+        run: bun run build
+
+      - name: Install runner dependencies
+        working-directory: devagent-runner
+        run: bun install
+
+      - name: Typecheck
+        working-directory: devagent-runner
+        run: bun run typecheck
+
+      - name: Test
+        working-directory: devagent-runner
+        run: bun run test

packages/adapters/src/index.test.ts

@@ -92,6 +92,44 @@ process.stdout.write(JSON.stringify({ protocolVersion: "0.1", type: "completed",
   assert.deepEqual(events.map((event) => event.type), ["started", "artifact", "completed"]);
 });
 
+test("DevAgentAdapter waits for close before reading result", async () => {
+  const { root, artifactDir, workspacePath } = await createWorkspace();
+  const stubPath = join(root, "devagent-close-stub.js");
+  await createStub(stubPath, `#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+const args = process.argv.slice(2);
+const requestPath = args[args.indexOf("--request") + 1];
+const artifactDir = args[args.indexOf("--artifact-dir") + 1];
+const request = JSON.parse(fs.readFileSync(requestPath, "utf8"));
+const artifactPath = path.join(artifactDir, "triage-report.md");
+const resultPath = path.join(artifactDir, "result.json");
+process.on("beforeExit", () => {
+  fs.writeFileSync(artifactPath, "# Triage\\n\\nLate output\\n");
+  fs.writeFileSync(resultPath, JSON.stringify({
+    protocolVersion: "0.1",
+    taskId: request.taskId,
+    status: "success",
+    artifacts: [{ kind: "triage-report", path: artifactPath, createdAt: new Date().toISOString() }],
+    metrics: { startedAt: new Date().toISOString(), finishedAt: new Date().toISOString(), durationMs: 1 }
+  }, null, 2));
+  process.stdout.write(JSON.stringify({ protocolVersion: "0.1", type: "started", at: new Date().toISOString(), taskId: request.taskId }) + "\\n");
+  process.stdout.write(JSON.stringify({ protocolVersion: "0.1", type: "artifact", at: new Date().toISOString(), taskId: request.taskId, artifact: { kind: "triage-report", path: artifactPath, createdAt: new Date().toISOString() } }) + "\\n");
+  process.stdout.write(JSON.stringify({ protocolVersion: "0.1", type: "completed", at: new Date().toISOString(), taskId: request.taskId, status: "success" }) + "\\n");
+});
+`);
+
+  const { events, result } = await collectEvents(
+    new DevAgentAdapter(`${process.execPath} ${stubPath}`),
+    createRequest("devagent"),
+    workspacePath,
+    artifactDir,
+  );
+
+  assert.equal(result.status, "success");
+  assert.deepEqual(events.map((event) => event.type), ["started", "artifact", "completed"]);
+});
+
 test("DevAgentAdapter reports cancelled runs as cancelled", async () => {
   const { root, artifactDir, workspacePath } = await createWorkspace();
   const stubPath = join(root, "devagent-cancel-stub.js");

packages/adapters/src/index.ts

@@ -16,6 +16,17 @@ import type {
 } from "@devagent-sdk/types";
 import { PROTOCOL_VERSION } from "@devagent-sdk/types";
 
+async function waitForFile(path: string, timeoutMs = 500): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() <= deadline) {
+    if (existsSync(path)) {
+      return true;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+  return existsSync(path);
+}
+
 function artifactFileName(kind: ArtifactKind): string {
   switch (kind) {
     case "triage-report":
@@ -243,12 +254,20 @@ class CliPromptAdapter implements ExecutorAdapter {
         reject(new RunnerError("PROCESS_LAUNCH_FAILED", error.message));
       });
 
-      child.once("exit", async (code: number | null, signal: NodeJS.Signals | null) => {
+      let exitCode: number | null = null;
+      let exitSignal: NodeJS.Signals | null = null;
+
+      child.once("exit", (code: number | null, signal: NodeJS.Signals | null) => {
+        exitCode = code;
+        exitSignal = signal;
+      });
+
+      child.once("close", async () => {
         try {
           const body = this.config.parseOutput
             ? await this.config.parseOutput(stdout, artifactDir)
             : stdout || stderr || `Executor ${this.id} completed without output.`;
-          const status = signal === "SIGTERM" ? "cancelled" : code === 0 ? "success" : "failed";
+          const status = exitSignal === "SIGTERM" ? "cancelled" : exitCode === 0 ? "success" : "failed";
           const result = await createFallbackResult(
             request,
             artifactDir,
@@ -345,18 +364,29 @@ export class DevAgentAdapter implements ExecutorAdapter {
 
     const resultPromise = new Promise<TaskExecutionResult>((resolve, reject) => {
       child.once("error", (error: Error) => reject(new RunnerError("PROCESS_LAUNCH_FAILED", error.message)));
-      child.once("exit", async (code: number | null, signal: NodeJS.Signals | null) => {
+      let exitCode: number | null = null;
+      let exitSignal: NodeJS.Signals | null = null;
+
+      child.once("exit", (code: number | null, signal: NodeJS.Signals | null) => {
+        exitCode = code;
+        exitSignal = signal;
+      });
+
+      child.once("close", async () => {
         try {
           const resultPath = join(artifactDir, "result.json");
-          if (!existsSync(resultPath)) {
-            const status = signal === "SIGTERM" ? "cancelled" : "failed";
+          if (!await waitForFile(resultPath)) {
+            const status = exitSignal === "SIGTERM" ? "cancelled" : "failed";
             const fallback = await createFallbackResult(
               request,
               artifactDir,
               status,
               startedAt,
               stderr || "DevAgent did not emit a result file.",
-              errorForStatus(status, signal === "SIGTERM" ? "Cancelled by operator" : (stderr || "Missing result.json")),
+              errorForStatus(
+                status,
+                exitSignal === "SIGTERM" ? "Cancelled by operator" : (stderr || "Missing result.json"),
+              ),
             );
             resolve(fallback);
             return;