update pytest config to add xdist / timeouts + ci improvements (#17)

* update pytest config to add xdist / timeouts

* wix ci

* update ci

---------

Co-authored-by: Tapan Chugh <tapanc@cs.washington.edu>

Author: chughtapan

.github/actions/setup-environment/action.yml

@@ -0,0 +1,17 @@
+name: Setup Environment
+description: Setup Python, uv, and install dev dependencies
+
+runs:
+  using: composite
+  steps:
+    - name: Setup uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.13"
+
+    - name: Install dependencies
+      shell: bash
+      run: uv pip install --system -e ".[dev]"

.github/workflows/ci.yml

@@ -6,51 +6,49 @@ on:
     branches: [main]
 
 jobs:
-  pre-commit:
+  lint:
     runs-on: ubuntu-latest
+    env:
+      UV_NO_SYNC: 1
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install dependencies
-        run: uv pip install --system -e ".[dev]"
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment
 
       - name: Install pre-commit
         run: uv pip install --system pre-commit
 
-      - name: Run pre-commit
+      - name: Run linting
         run: SKIP=mypy pre-commit run --all-files
 
+  typecheck:
+    runs-on: ubuntu-latest
+    env:
+      UV_NO_SYNC: 1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment
+
       - name: Run mypy
         run: uv run mypy src/ tests/ --exclude 'tests/benchmarks/appworld/'
 
   test:
     runs-on: ubuntu-latest
     env:
       CI: 1
+      UV_NO_SYNC: 1
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install dependencies
-        run: uv pip install --system -e ".[dev]"
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment
 
       - name: Run unit tests
         run: uv run pytest tests/unit/ -v -p no:warnings

docs/evals.md

@@ -87,6 +87,21 @@ Validate existing logs without running new tests:
 .venv/bin/pytest tests/benchmarks/bfcl/test_bfcl.py --validate-only --log-dir outputs/experiment1/raw
 ```
 
+### Parallel Execution
+
+Run tests in parallel using multiple workers:
+
+```bash
+# Run with 4 workers
+.venv/bin/pytest tests/benchmarks/bfcl/test_bfcl.py -n 4
+
+# Run with 8 workers
+.venv/bin/pytest tests/benchmarks/appworld/test_appworld.py --dataset train -n 8
+
+# Auto-detect number of CPUs
+.venv/bin/pytest tests/benchmarks/bfcl/test_bfcl.py -n auto
+```
+
 ## Further Reading
 
 ### BFCL

pyproject.toml

@@ -27,6 +27,8 @@ where = ["src"]
 dev = [
     "pytest>=7.0",
     "pytest-asyncio>=0.21",
+    "pytest-xdist>=3.0",
+    "pytest-timeout>=2.0",
     "ruff",
     "mypy",
 ]
@@ -86,6 +88,8 @@ filterwarnings = [
     "ignore::sqlalchemy.exc.SADeprecationWarning:appworld.apps.lib.models.db",
     "ignore::pydantic.warnings.PydanticDeprecatedSince20:appworld.apps.*",
 ]
+timeout = 0
+timeout_method = "thread"  # Use thread-based timeout for better async compatibility
 
 [tool.mypy]
 warn_return_any = true

tests/benchmarks/appworld/README.md

@@ -54,6 +54,15 @@ pytest tests/benchmarks/appworld/test_appworld.py --validate-only
 --temperature 0.001  # Temperature for sampling (default: 0.001)
 ```
 
+### Parallel Execution
+```bash
+-n 4     # Run with 4 workers
+-n 8     # Run with 8 workers
+-n auto  # Auto-detect number of CPUs
+```
+
+Example: `pytest tests/benchmarks/appworld/test_appworld.py --dataset train -n 4`
+
 ## File Structure
 
 ```

tests/benchmarks/appworld/test_appworld.py

@@ -50,6 +50,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
 
 
 @pytest.mark.asyncio
+@pytest.mark.timeout(300)
 async def test_appworld(
     task_id: str,
     model: str,

tests/conftest.py

@@ -50,7 +50,6 @@ def pytest_addoption(parser: pytest.Parser) -> None:
     parser.addoption("--output-dir", default="outputs", help="Output directory for results")
     parser.addoption("--validate-only", action="store_true", help="Only validate existing logs")
     parser.addoption("--log-dir", default="outputs/raw", help="Directory with logs (for validate mode)")
-    parser.addoption("--max-workers", default=4, type=int, help="Max concurrent tests (default: 4)")
 
 
 def pytest_configure(config: pytest.Config) -> None: