Merge pull request #132 from TransformerOptimus/feat/eval-harness-bench-runner

Eval harness: bench-runner + context-sync + bulk-indexer (+CI)

Author: adithyn7

.cargo/config.toml

@@ -0,0 +1,10 @@
+# The static musl bench-runner must be a NON-PIE static binary.
+#
+# Rust defaults the x86_64-unknown-linux-musl target to static-PIE, which
+# segfaults at startup on some native x86_64 hosts (it only ran under emulation
+# in local testing; CI on a native runner caught the SIGSEGV). relocation-model=
+# static produces a classic, position-dependent, fully static executable that
+# runs everywhere — including the SWE-bench task containers the bench-runner
+# ships into.
+[target.x86_64-unknown-linux-musl]
+rustflags = ["-C", "relocation-model=static"]

.github/workflows/ci.yml

@@ -48,6 +48,48 @@ jobs:
       - name: cargo check (workspace)
         run: cargo check --workspace --all-targets
 
+      - name: cargo test (workspace)
+        run: cargo test --workspace --locked
+
+  bench-runner-musl:
+    # The bench-runner ships as a static musl binary into toolchain-free task
+    # containers. cargo check can't catch a native-tls regression (it only breaks
+    # the musl link), so we build it for real here to guard the rustls-only (R1)
+    # invariant. Builds ONLY -p bench-runner, so no tauri/frontend deps are needed.
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: x86_64-unknown-linux-musl
+
+      - uses: swatinem/rust-cache@v2
+        with:
+          key: musl-bench-runner
+
+      - name: Install musl toolchain
+        run: sudo apt-get update && sudo apt-get install -y musl-tools
+
+      - name: Build static bench-runner
+        env:
+          CC_x86_64_unknown_linux_musl: musl-gcc
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc
+        run: cargo build --release --locked -p bench-runner --target x86_64-unknown-linux-musl
+
+      - name: Assert static & self-contained (R1 guard)
+        run: |
+          BIN=target/x86_64-unknown-linux-musl/release/bench-runner
+          file "$BIN"
+          ldd "$BIN" 2>&1 || true
+          # musl static-pie resolves NO shared libs; a real dynamic dep shows "=>".
+          # (file's wording for static-pie varies by version, so don't grep file.)
+          if ldd "$BIN" 2>&1 | grep -qE '=>|libssl|libcrypto'; then
+            echo "ERROR: dynamic dependency present (native-tls regressed?)"; exit 1
+          fi
+          # Self-contained: runs on this host even though no musl libc is installed.
+          "$BIN" --help >/dev/null && echo "runs standalone: OK"
+
   frontend:
     runs-on: ubuntu-latest
     steps:

.github/workflows/release.yml

@@ -105,3 +105,46 @@ jobs:
           name: supercoder-${{ matrix.target }}
           path: ${{ join(fromJSON(steps.tauri.outputs.artifactPaths), '\n') }}
           if-no-files-found: ignore
+
+  bench-runner:
+    # The eval-harness bench-runner: a static musl x86_64 binary that runs the
+    # agent headless inside toolchain-free task containers. Not a tauri bundle, so
+    # it lives outside the desktop build matrix.
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: x86_64-unknown-linux-musl
+
+      - uses: swatinem/rust-cache@v2
+        with:
+          key: musl-bench-runner
+
+      - name: Install musl toolchain
+        run: sudo apt-get update && sudo apt-get install -y musl-tools
+
+      - name: Build static bench-runner
+        env:
+          CC_x86_64_unknown_linux_musl: musl-gcc
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc
+        run: cargo build --release --locked -p bench-runner --target x86_64-unknown-linux-musl
+
+      - name: Stage artifact
+        run: cp target/x86_64-unknown-linux-musl/release/bench-runner bench-runner-x86_64-unknown-linux-musl
+
+      - name: Upload to release
+        if: github.event_name == 'release'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload "${{ github.event.release.tag_name }}" bench-runner-x86_64-unknown-linux-musl --clobber
+
+      - name: Upload as workflow artifact (workflow_dispatch only)
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-runner-x86_64-unknown-linux-musl
+          path: bench-runner-x86_64-unknown-linux-musl