feat(results): periodic WIP checkpoints for in-progress eval runs (#1369)

* feat(results): periodic WIP checkpoints for in-progress eval runs

Every ~30s, force-pushes partial run output to a unique
`agentv/inflight/<hostname>/<run-dir>` branch in the configured results
repository. On successful run completion the final results are published
to the normal results branch and the WIP branch is deleted. On failure
or abort, the WIP branch is left intact for manual recovery.

Implementation:
- `buildWipBranchName` / `setupWipWorktree` / `pushWipCheckpoint` /
  `deleteWipBranch` helpers in `packages/core/src/evaluation/results-repo.ts`
- `WipCheckpointLoop` class in `apps/cli/src/commands/eval/wip-checkpoint.ts`
  uses `setInterval(...).unref()` so the timer never prevents process exit
- Loop wired into `run-eval.ts`; gated on `auto_push` config flag
- All checkpoint errors are best-effort (logged as warnings, never propagate)

Closes #av-o3w

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix(results): guard wip checkpoint cleanup

* fix(results): base checkpoints on storage branch

* fix(results): configure git identity for remote commits

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: christso

apps/cli/src/commands/eval/run-eval.ts

@@ -32,7 +32,12 @@ import {
 } from '@agentv/core';
 
 import { enforceRequiredVersion } from '../../version-check.js';
-import { maybeAutoExportRunArtifacts } from '../results/remote.js';
+import {
+  type RemoteExportStatus,
+  getRelativeRunPath,
+  loadNormalizedResultsConfig,
+  maybeAutoExportRunArtifacts,
+} from '../results/remote.js';
 import {
   aggregateRunDir,
   buildTestTargetKey,
@@ -60,6 +65,7 @@ import {
 } from './statistics.js';
 import { type TargetSelection, selectMultipleTargets, selectTarget } from './targets.js';
 import type { TaskBundleTargetSelection } from './task-bundle.js';
+import { WipCheckpointLoop } from './wip-checkpoint.js';
 
 const DEFAULT_WORKERS = 3;
 
@@ -1543,6 +1549,24 @@ export async function runEvalCommand(
     });
   }
 
+  // Periodic WIP checkpoint loop: push partial results to a unique non-default
+  // branch every ~60s so pod loss doesn't discard completed-test output.
+  // Only active when a results repo with auto_push is configured; otherwise a no-op.
+  let wipLoop: WipCheckpointLoop | undefined;
+  let wipCleanedUp = false;
+  let finalExportStatus: RemoteExportStatus = 'disabled';
+  {
+    const wipConfig = await loadNormalizedResultsConfig(cwd).catch(() => undefined);
+    if (wipConfig?.auto_push) {
+      wipLoop = new WipCheckpointLoop({
+        config: wipConfig,
+        runDir,
+        destinationPath: getRelativeRunPath(cwd, runDir),
+      });
+      await wipLoop.start();
+    }
+  }
+
   // Eval files run sequentially; within each file, --workers N test cases run in parallel.
   // This matches industry practice (promptfoo, deepeval, OpenAI Evals) and avoids cross-file
   // workspace races without any grouping complexity.
@@ -1825,7 +1849,7 @@ export async function runEvalCommand(
       // Persist last run path for `agentv results` commands
       await saveRunCache(cwd, outputPath).catch(() => undefined);
 
-      await maybeAutoExportRunArtifacts({
+      finalExportStatus = await maybeAutoExportRunArtifacts({
         cwd,
         run_dir: runDir,
         test_files: activeTestFiles,
@@ -1873,6 +1897,17 @@ export async function runEvalCommand(
       );
     }
 
+    // WIP cleanup on success: remove the WIP branch only after the final
+    // results branch is confirmed published (or confirmed already up to date).
+    // If export failed, leave the remote WIP checkpoint as the durable copy.
+    if (
+      wipLoop &&
+      (finalExportStatus === 'published' || finalExportStatus === 'already_published')
+    ) {
+      wipCleanedUp = true;
+      await wipLoop.stopAndDeleteWipBranch();
+    }
+
     return {
       executionErrorCount: summary.executionErrorCount,
       outputPath,
@@ -1883,6 +1918,11 @@ export async function runEvalCommand(
       budgetExceeded: runBudgetExceeded || undefined,
     };
   } finally {
+    // WIP cleanup on failure/interrupt: stop the loop but leave the remote
+    // WIP branch intact for manual recovery.
+    if (wipLoop && !wipCleanedUp) {
+      await wipLoop.stop().catch(() => undefined);
+    }
     unsubscribeCodexLogs();
     unsubscribePiLogs();
     unsubscribeCopilotSdkLogs();

apps/cli/src/commands/eval/wip-checkpoint.ts

@@ -0,0 +1,146 @@
+/**
+ * WIP (work-in-progress) checkpoint loop for in-progress eval runs.
+ *
+ * Periodically force-pushes the partial run output directory to a unique
+ * non-default branch (`agentv/inflight/<hostname>/<run-dir-basename>`) in the
+ * configured results repository. This protects against pod/process loss by
+ * keeping completed-test results durable without requiring PVC or S3.
+ *
+ * Branch lifecycle:
+ *   1. Start:   set up a persistent git worktree for the WIP branch.
+ *   2. Running: every ~30s, copy run dir → worktree, amend-commit, force-push.
+ *   3. Success: after final publish to the normal results branch, delete the WIP branch.
+ *   4. Failure: leave the WIP branch for manual recovery.
+ *
+ * Manual recovery from a WIP branch:
+ *   git clone <results-repo> /tmp/recovery
+ *   cd /tmp/recovery && git checkout agentv/inflight/<hostname>/<run-dir>
+ *   cp -r .agentv/results/runs/<run-dir> <project>/.agentv/results/runs/
+ *   agentv eval <eval-file> --output <project>/.agentv/results/runs/<run-dir> --resume
+ *
+ * All checkpoint operations are best-effort: failures are logged as warnings
+ * and never propagate to the eval run.
+ */
+
+import {
+  type NormalizedResultsConfig,
+  type WipWorktreeHandle,
+  buildWipBranchName,
+  deleteWipBranch,
+  pushWipCheckpoint,
+  setupWipWorktree,
+} from '@agentv/core';
+
+const WIP_CHECKPOINT_INTERVAL_MS = 30_000;
+
+export interface WipCheckpointLoopDependencies {
+  readonly buildWipBranchName: (runDir: string) => string;
+  readonly deleteWipBranch: typeof deleteWipBranch;
+  readonly pushWipCheckpoint: typeof pushWipCheckpoint;
+  readonly setupWipWorktree: typeof setupWipWorktree;
+}
+
+const defaultDependencies: WipCheckpointLoopDependencies = {
+  buildWipBranchName,
+  deleteWipBranch,
+  pushWipCheckpoint,
+  setupWipWorktree,
+};
+
+function warnCheckpointError(context: string, error: unknown): void {
+  const message = error instanceof Error ? error.message : String(error);
+  console.warn(`WIP checkpoint: ${context}: ${message}`);
+}
+
+export class WipCheckpointLoop {
+  readonly wipBranch: string;
+  private readonly config: NormalizedResultsConfig;
+  private readonly runDir: string;
+  private readonly destinationPath: string;
+  private readonly intervalMs: number;
+  private readonly deps: WipCheckpointLoopDependencies;
+  private handle: WipWorktreeHandle | undefined;
+  private timer: ReturnType<typeof setInterval> | undefined;
+  private checkpointInFlight: Promise<void> | undefined;
+  private active = false;
+
+  constructor(params: {
+    readonly config: NormalizedResultsConfig;
+    readonly runDir: string;
+    readonly destinationPath: string;
+    readonly intervalMs?: number;
+    readonly dependencies?: WipCheckpointLoopDependencies;
+  }) {
+    this.config = params.config;
+    this.runDir = params.runDir;
+    this.destinationPath = params.destinationPath;
+    this.intervalMs = params.intervalMs ?? WIP_CHECKPOINT_INTERVAL_MS;
+    this.deps = params.dependencies ?? defaultDependencies;
+    this.wipBranch = this.deps.buildWipBranchName(params.runDir);
+  }
+
+  async start(): Promise<void> {
+    try {
+      this.handle = await this.deps.setupWipWorktree({
+        config: this.config,
+        wipBranch: this.wipBranch,
+      });
+    } catch (err) {
+      warnCheckpointError('failed to set up WIP worktree', err);
+      return;
+    }
+    this.active = true;
+    this.timer = setInterval(() => {
+      this.runCheckpointIfIdle();
+    }, this.intervalMs);
+    // Unref so the timer never prevents process exit.
+    this.timer.unref?.();
+  }
+
+  private runCheckpointIfIdle(): void {
+    if (!this.active || this.checkpointInFlight) return;
+    this.checkpointInFlight = this.checkpoint()
+      .catch((err) => warnCheckpointError('push failed', err))
+      .finally(() => {
+        this.checkpointInFlight = undefined;
+      });
+  }
+
+  private async checkpoint(): Promise<void> {
+    if (!this.handle) return;
+    await this.deps.pushWipCheckpoint({
+      handle: this.handle,
+      sourceDir: this.runDir,
+      destinationPath: this.destinationPath,
+    });
+  }
+
+  /** Stop the loop and clean up the local worktree. Does NOT delete the remote WIP branch. */
+  async stop(): Promise<void> {
+    this.active = false;
+    if (this.timer !== undefined) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    await this.checkpointInFlight;
+    if (this.handle) {
+      await this.handle
+        .cleanup()
+        .catch((err) => warnCheckpointError('worktree cleanup failed', err));
+      this.handle = undefined;
+    }
+  }
+
+  /**
+   * Stop the loop and delete the remote WIP branch.
+   * Call after a successful run to keep the results repo tidy.
+   */
+  async stopAndDeleteWipBranch(): Promise<void> {
+    await this.stop();
+    try {
+      await this.deps.deleteWipBranch({ config: this.config, wipBranch: this.wipBranch });
+    } catch (err) {
+      warnCheckpointError(`failed to delete remote branch ${this.wipBranch}`, err);
+    }
+  }
+}

apps/cli/src/commands/results/remote.ts

@@ -102,6 +102,8 @@ export interface RemoteExportPayload {
   readonly experiment?: string;
 }
 
+export type RemoteExportStatus = 'disabled' | 'published' | 'already_published' | 'failed';
+
 export interface RemoteResultsStatus extends ResultsRepoStatus {
   readonly run_count: number;
 }
@@ -120,7 +122,7 @@ function statusForResult(result: EvaluationResult): 'PASS' | 'FAIL' | 'ERROR' {
   return result.score >= DEFAULT_THRESHOLD ? 'PASS' : 'FAIL';
 }
 
-function getRelativeRunPath(cwd: string, runDir: string): string {
+export function getRelativeRunPath(cwd: string, runDir: string): string {
   const relative = path.relative(path.join(cwd, '.agentv', 'results', 'runs'), runDir);
   if (!relative.startsWith('..') && !path.isAbsolute(relative)) {
     return relative;
@@ -150,7 +152,7 @@ async function maybeWarnLargeArtifact(runDir: string): Promise<void> {
   }
 }
 
-async function loadNormalizedResultsConfig(
+export async function loadNormalizedResultsConfig(
   cwd: string,
   projectId?: string,
 ): Promise<NormalizedResultsConfig | undefined> {
@@ -427,10 +429,12 @@ export async function clearRemoteRunTags(
   return deleteRemoteRunTags(config.path, meta.path);
 }
 
-export async function maybeAutoExportRunArtifacts(payload: RemoteExportPayload): Promise<void> {
+export async function maybeAutoExportRunArtifacts(
+  payload: RemoteExportPayload,
+): Promise<RemoteExportStatus> {
   const config = await loadNormalizedResultsConfig(payload.cwd);
   if (!config?.auto_push) {
-    return;
+    return 'disabled';
   }
 
   try {
@@ -448,12 +452,14 @@ export async function maybeAutoExportRunArtifacts(payload: RemoteExportPayload):
 
     if (!pushed) {
       console.warn('Warning: results export produced no git changes. Skipping push.');
-      return;
+      return 'already_published';
     }
 
     console.log(`Results pushed to ${config.repo} (${config.path}/${relativeRunPath})`);
+    return 'published';
   } catch (error) {
     console.warn(`Warning: skipping results export: ${getStatusMessage(error)}`);
     console.warn("Warning: Run 'gh auth login' if GitHub authentication is missing.");
+    return 'failed';
   }
 }

apps/cli/test/commands/eval/wip-checkpoint.test.ts

@@ -0,0 +1,88 @@
+import { afterEach, describe, expect, it, mock } from 'bun:test';
+
+import { WipCheckpointLoop } from '../../../src/commands/eval/wip-checkpoint.js';
+
+type Deferred<T> = {
+  readonly promise: Promise<T>;
+  readonly resolve: (value: T) => void;
+  readonly reject: (error: unknown) => void;
+};
+
+function deferred<T>(): Deferred<T> {
+  let resolve!: (value: T) => void;
+  let reject!: (error: unknown) => void;
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return { promise, resolve, reject };
+}
+
+async function waitFor(predicate: () => boolean): Promise<void> {
+  const deadline = Date.now() + 500;
+  while (!predicate()) {
+    if (Date.now() > deadline) {
+      throw new Error('Timed out waiting for condition');
+    }
+    await new Promise((resolve) => setTimeout(resolve, 5));
+  }
+}
+
+const cleanupMock = mock(async () => {});
+const deleteWipBranchMock = mock(async () => {});
+let pushWipCheckpointImplementation = async () => true;
+const pushWipCheckpointMock = mock(async () => pushWipCheckpointImplementation());
+
+afterEach(() => {
+  cleanupMock.mockClear();
+  deleteWipBranchMock.mockClear();
+  pushWipCheckpointMock.mockClear();
+  pushWipCheckpointImplementation = async () => true;
+});
+
+describe('WipCheckpointLoop', () => {
+  it('waits for an in-flight checkpoint before cleanup and remote branch deletion', async () => {
+    const checkpoint = deferred<boolean>();
+    pushWipCheckpointImplementation = async () => checkpoint.promise;
+
+    const loop = new WipCheckpointLoop({
+      config: {
+        mode: 'github',
+        repo: 'https://github.com/example/results.git',
+        branch: 'main',
+        path: '/tmp/results',
+        auto_push: true,
+        branch_prefix: 'agentv/results',
+      },
+      runDir: '/tmp/run-001',
+      destinationPath: 'default/run-001',
+      intervalMs: 1,
+      dependencies: {
+        buildWipBranchName: (runDir) => `agentv/inflight/test/${runDir.split('/').pop()}`,
+        deleteWipBranch: deleteWipBranchMock,
+        pushWipCheckpoint: pushWipCheckpointMock,
+        setupWipWorktree: mock(async ({ wipBranch }) => ({
+          wipBranch,
+          worktreeDir: '/tmp/wip-worktree',
+          cloneDir: '/tmp/wip-clone',
+          cleanup: cleanupMock,
+        })),
+      },
+    });
+
+    await loop.start();
+    await waitFor(() => pushWipCheckpointMock.mock.calls.length === 1);
+
+    const stopped = loop.stopAndDeleteWipBranch();
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(cleanupMock).not.toHaveBeenCalled();
+    expect(deleteWipBranchMock).not.toHaveBeenCalled();
+
+    checkpoint.resolve(true);
+    await stopped;
+
+    expect(cleanupMock).toHaveBeenCalledTimes(1);
+    expect(deleteWipBranchMock).toHaveBeenCalledTimes(1);
+  });
+});