fix(evals): close inherited stdin in SWE-bench sandboxes (#2303)

Backport SWE-ReX #281 semantics at the mini-swe-agent Modal adapter so stdin-reading commands receive EOF without breaking pipes or heredocs. Keep the existing 300-second command timeout and add regression coverage.

中文：修复 SWE-bench 沙箱继承服务端标准输入后，cat、grep 等命令无输入时持续阻塞的问题。在 mini-swe-agent Modal 适配层回移 SWE-ReX #281 的语义，使此类命令立即收到 EOF，同时保留管道和 heredoc 行为；维持现有 300 秒命令超时并补充回归测试。

Author: adibarra

benchmarks/benchmark_lib.sh

@@ -1113,7 +1113,7 @@ _install_swebench_agent_deps() {
     python3 -m pip install -q --no-cache-dir --break-system-packages \
         'mini-swe-agent==2.4.5' 'swe-rex[modal]==1.4.0' || true
     _patch_swebench_agent || \
-        echo "WARN: mini-swe-agent/swe-rex patches failed; sandboxes will leak until runtime_timeout and budget-exhausted instances will submit nothing" >&2
+        echo "WARN: mini-swe-agent/swe-rex patches failed; sandbox cleanup, submission fallback, or non-interactive stdin handling may be degraded" >&2
 }
 
 _patch_swebench_agent() {

utils/evals/EVALS.md

@@ -174,7 +174,8 @@ Source rewrites are anchor-checked, idempotent, and atomic.
   compatibility (no `{"type": "text"}` injection for non-HF tokenizers).
   Copied into a temp dir as `sitecustomize.py` on `PYTHONPATH`.
 - `patch_swebench_agent.py` (`_patch_swebench_agent`): mini-swe-agent/swe-rex
-  sandbox lifecycle cleanup + budget-exhaustion submission fallback.
+  sandbox lifecycle cleanup, budget-exhaustion submission fallback, and the
+  [SWE-ReX #281](https://github.com/SWE-agent/SWE-ReX/pull/281) closed-stdin fix.
 - `patch_swebench_scoring.py` (`_patch_swebench_scoring`): swebench Modal
   scorer reserved-CPU reduction + sandbox termination on instance completion.
 
@@ -199,8 +200,9 @@ append_lm_eval_summary
   local endpoint, each instance's shell running in a Modal sandbox via swe-rex — the real
   SWE-bench setting) or `single-shot` (lm-eval, one prompt per instance — a ~10% floor baseline,
   kept only as an explicit debugging escape hatch). Agentic knobs: `SWEBENCH_AGENT_WORKERS`
-  (default: the config's `CONC`, else 64), `SWEBENCH_AGENT_STEP_LIMIT` (75), `SWEBENCH_AGENT_TIMEOUT`
-  (4h), `SWEBENCH_AGENT_SANDBOX_CPU` (unset = Modal default), `SWEBENCH_MODAL_APP_NAME`
+  (default: the config's `CONC`, else 64), `SWEBENCH_AGENT_STEP_LIMIT` (75),
+  `SWEBENCH_AGENT_CMD_TIMEOUT` (per command, 300s), `SWEBENCH_AGENT_TIMEOUT` (4h),
+  `SWEBENCH_AGENT_SANDBOX_CPU` (unset = Modal default), and `SWEBENCH_MODAL_APP_NAME`
   (`infx-evals-swe`).
 - Run size: `EVAL_LIMIT` empty runs the full ~300-instance split; a positive integer runs the
   first N as an explicit smoke-test slice. `EVAL_LIMIT=full` (or `0`) also selects the full split.

utils/evals/patches/patch_swebench_agent.py

@@ -40,9 +40,27 @@ def _patch(path: str, replacements: list[tuple[str, str, str]], label: str) -> b
     return True
 
 
+
+def _patch_swerex_environment(path: str) -> bool:
+    return _patch(
+        path,
+        [
+            (
+                '        command = action.get("command", "") if isinstance(action, dict) else action\n'
+                "        try:",
+                '        command = action.get("command", "") if isinstance(action, dict) else action\n'
+                '        command = f"exec </dev/null\\n{command}"  # close inherited server stdin (SWE-ReX #281)\n'
+                "        try:",
+                "# close inherited server stdin (SWE-ReX #281)",
+            ),
+        ],
+        "swebench-agentic",
+    )
+
 def main() -> int:
     import minisweagent.run.benchmarks.swebench as mini_swebench
     import swerex.deployment.modal as rex_modal
+    import minisweagent.environments.extra.swerex_modal as mini_rex_modal
 
     mini_ok = _patch(
         mini_swebench.__file__,
@@ -100,6 +118,8 @@ def main() -> int:
         ],
         "swebench-agentic",
     )
+    mini_env_ok = _patch_swerex_environment(mini_rex_modal.__file__)
+
 
     app_name = os.environ.get("SWEBENCH_MODAL_APP_NAME", "infx-evals-swe")
     rex_ok = _patch(
@@ -137,7 +157,7 @@ def main() -> int:
         ],
         "swebench-agentic",
     )
-    return 0 if mini_ok and rex_ok else 1
+    return 0 if mini_ok and mini_env_ok and rex_ok else 1
 
 
 if __name__ == "__main__":

utils/evals/test_eval_patches.py

@@ -1,6 +1,7 @@
 import importlib.util
 import json
 import runpy
+import subprocess
 import sys
 import types
 from pathlib import Path
@@ -42,6 +43,67 @@ def test_agent_patch_is_atomic_and_idempotent(tmp_path):
     assert target.read_text() == patched
 
 
+def test_agent_patch_closes_inherited_stdin(tmp_path):
+    target = tmp_path / "swerex_modal.py"
+    target.write_text(
+        """import json
+import subprocess
+
+class Environment:
+    def execute(self, action):
+        command = action.get("command", "") if isinstance(action, dict) else action
+        try:
+            result = subprocess.run(
+                command,
+                shell=True,
+                timeout=0.2,
+                stdout=subprocess.PIPE,
+                text=True,
+            )
+        except subprocess.TimeoutExpired:
+            return {"returncode": -1, "output": "timed out"}
+        return {"returncode": result.returncode, "output": result.stdout}
+
+environment = Environment()
+commands = ["cat", "printf 'pipe-ok\\\\n' | cat"]
+print(json.dumps([environment.execute({"command": command}) for command in commands]))
+"""
+    )
+
+    assert agent_patch._patch_swerex_environment(str(target))
+    patched = target.read_text()
+    assert agent_patch._patch_swerex_environment(str(target))
+    assert target.read_text() == patched
+
+    process = subprocess.Popen(
+        [sys.executable, str(target)],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        assert process.wait(timeout=2) == 0
+    finally:
+        if process.poll() is None:
+            process.kill()
+            process.wait()
+        assert process.stdin is not None
+        process.stdin.close()
+
+    assert process.stdout is not None
+    assert process.stderr is not None
+    output = process.stdout.read()
+    errors = process.stderr.read()
+    process.stdout.close()
+    process.stderr.close()
+    assert not errors
+    assert json.loads(output) == [
+        {"returncode": 0, "output": ""},
+        {"returncode": 0, "output": "pipe-ok\n"},
+    ]
+
+
 def test_scoring_patch_is_atomic_and_idempotent(tmp_path):
     target = tmp_path / "run_evaluation_modal.py"
     target.write_text("prefix\ncpu=4,\nsuffix\n")