From 226e8286a3f46e66b43e12cde87a5eaea255a92f Mon Sep 17 00:00:00 2001
From: Shehab Yasser <shehab.yasser@scale.com>
Date: Tue, 28 Jul 2026 18:33:52 +0300
Subject: [PATCH] ci: run pytest for vero and the six baseline agents

Nothing in this repository ran tests. The only checks on a pull request were
Greptile Review and Socket Security, neither of which invokes pytest, and there
was no .github directory on any branch. #61 sat open with a red target suite
because of it: the change there reaches the OpenAI client through
`with_options`, the test fake did not implement it, and
`test_agent_submits_response_and_populates_context` had been failing on
AttributeError since the PR was opened with nothing to surface it.

Two jobs, seven suites, ~450 vero tests plus 20 across the agents:

- `vero`: uv sync --locked, then pytest. Two placeholder credentials are set
  because the task compiler refuses to emit a task whose declared credentials
  are absent from the environment. Without them five build tests fail on
  "declared task credentials are missing", which reads like a code failure and
  is not one. Setting VERO_SKIP_SECRET_CHECK instead would be wrong: it makes
  test_compiler_checks_secrets_before_writing_and_rejects_source_overlap
  vacuous, since that test asserts the check fires.
- `targets`: one job per baseline agent, fail-fast off so a single broken agent
  does not mask the other five.

Verified locally at Python 3.12 against ed55bc7 before committing: vero 447
passed / 11 skipped, browsecomp-plus 3, gaia 2, officeqa 2, swe-atlas-qna 2,
swe-bench-pro 3, tau3 8. `uv lock --check` is clean in all seven projects, so
--locked will not spuriously fail.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .github/workflows/tests.yml | 89 +++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 .github/workflows/tests.yml

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 00000000..9d4f3cee
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,89 @@
+name: tests
+
+# Nothing in this repository ran tests before this workflow: the only checks on a
+# pull request were Greptile Review and Socket Security, neither of which invokes
+# pytest. PR #61 sat open with a red target suite as a result.
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Every package here resolves from public PyPI (vero/pyproject.toml pins that
+  # index explicitly), so no private-registry credentials are needed. Pinned so a
+  # runner image bump cannot silently change the interpreter: vero requires
+  # >=3.11,<3.14 and the six targets require >=3.12.
+  UV_PYTHON: "3.12"
+
+jobs:
+  vero:
+    name: vero
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    env:
+      # The task compiler refuses to emit a task whose declared credentials are
+      # absent ("declared task credentials are missing", compiler.py), so the
+      # build tests need these present. Placeholders only: nothing dials them,
+      # and port 1 fails fast instead of hanging if anything ever tries.
+      OPENAI_API_KEY: ci-not-a-real-key
+      OPENAI_BASE_URL: http://127.0.0.1:1/v1
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Install dependencies
+        working-directory: vero
+        # --locked fails rather than relocking, so a pyproject edit that forgets
+        # to update uv.lock is a visible error instead of a silent resolve.
+        run: uv sync --locked
+
+      - name: Run tests
+        working-directory: vero
+        run: uv run pytest -q
+
+  targets:
+    # The six editable baseline agents. Each is its own uv project with its own
+    # lock, so they get one job apiece rather than a shared environment.
+    name: ${{ matrix.benchmark }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      # One broken agent should not mask the state of the other five.
+      fail-fast: false
+      matrix:
+        benchmark:
+          - browsecomp-plus
+          - gaia
+          - officeqa
+          - swe-atlas-qna
+          - swe-bench-pro
+          - tau3
+    steps:
+      # No `submodules: recursive`: the only submodule is the BrowseComp-Plus
+      # upstream corpus, which no test reads.
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Install dependencies
+        working-directory: harness-engineering-bench/${{ matrix.benchmark }}/baseline/target
+        run: uv sync --locked
+
+      - name: Run tests
+        working-directory: harness-engineering-bench/${{ matrix.benchmark }}/baseline/target
+        run: uv run pytest -q