[WIP]patcheval env

.gitignore

@@ -21,3 +21,6 @@ tmp/
 
 # External SDK checkouts should be installed as dependencies instead.
 /wt-data-gateway/
+
+#
+wheels/

clusters/docker_clusters.py

@@ -527,18 +527,29 @@ async def _install_runner_script_if_needed(
         runner_host_path = self._resolve_runner_host_path(docker_cfg.get("runner_script_host_path"))
         if runner_host_path is None:
             runner_host_path = self._runner_host_path
+        runner_container_path = str(
+            docker_cfg.get("runner_container_path", self._runner_container_path) or self._runner_container_path
+        ).strip()
         if runner_host_path is None or not runner_host_path.is_file():
-            raise RuntimeError(f"OpenClaw runner script not found: {runner_host_path}")
+            raise RuntimeError(f"Safactory runner script not found: {runner_host_path}")
+        if not runner_container_path:
+            raise RuntimeError("Safactory runner container path is empty")
 
+        runner_parent = str(Path(runner_container_path).parent)
         await self._run_required(
-            [self.docker_bin, "cp", str(runner_host_path), f"{record.container_id}:{self._runner_container_path}"],
-            action=f"install Safactory OpenClaw runner in {record.container_name}",
+            [self.docker_bin, "exec", record.container_id, "mkdir", "-p", runner_parent],
+            action=f"prepare runner directory in {record.container_name}",
+            timeout_s=self._start_timeout_s,
+        )
+        await self._run_required(
+            [self.docker_bin, "cp", str(runner_host_path), f"{record.container_id}:{runner_container_path}"],
+            action=f"install Safactory runner in {record.container_name}",
             timeout_s=self._start_timeout_s,
         )
         log.debug(
-            "installed Safactory OpenClaw runner into %s:%s",
+            "installed Safactory runner into %s:%s",
             record.container_name,
-            self._runner_container_path,
+            runner_container_path,
         )
 
     async def _start_container(self, *, env_name: str, image: str) -> DockerContainerRecord:
@@ -562,9 +573,12 @@ async def _start_container(self, *, env_name: str, image: str) -> DockerContaine
         )
         resolved_max_runs = int(docker_cfg.get("max_runs_per_container", self._default_max_runs) or 0)
         reuse_container = self._resolve_reuse(docker_cfg)
+        runner_container_path = str(
+            docker_cfg.get("runner_container_path", self._runner_container_path) or self._runner_container_path
+        ).strip()
         install_runner = docker_cfg.get("install_runner_script")
         if install_runner is None:
-            install_runner = self._runner_container_path in resolved_run_command
+            install_runner = runner_container_path in resolved_run_command
         run_cmd = self._build_run_command(
             image=image,
             name=container_name,
@@ -610,7 +624,7 @@ async def _start_container(self, *, env_name: str, image: str) -> DockerContaine
                     "remove_timeout_s": self._remove_timeout_s,
                     "remove_retries": self._remove_retries,
                     "remove_retry_delay_s": self._remove_retry_delay_s,
-                    "runner_container_path": self._runner_container_path,
+                    "runner_container_path": runner_container_path,
                     "install_runner_script": bool(install_runner),
                 }
             ),

env/geo3k/geo3k_config.yaml

@@ -3,7 +3,7 @@ environments:
     env_image: safactory-geo3k:py311
     env_num: 1
     # chenhegu/geo3k_imgurl train split at the pinned Hugging Face snapshot.
-    dataset: /mnt/shared-storage-user/yinzhenyun/datasets--chenhegu--geo3k_imgurl/snapshots/a924f868cefdf3b8306196b04edaa88d930978e5/train.parquet
+    dataset: /mnt/shared-storage-user/leishanzhe/dataset/chenhegu/geo3k_imgurl/train.parquet
     dataset_load_mode: parquet_row_ref
     # Exclude preprocessed_images: it contains binary image bytes and the runner
     # only consumes the data URLs already stored in images.

env/patcheval/datasets/cve-2020-25459.jsonl

@@ -0,0 +1 @@
+{"cve_id":"CVE-2020-25459","work_dir":"/workspace/FATE","problem_statement":"Fix CVE-2020-25459 in FATE. sync_tree sends sensitive decision-tree information to hosts during federated training. Remove sensitive information before transfer while preserving training behavior.\nVulnerability Location\n[{\"file_path\":\"federatedml/tree/hetero/hetero_decision_tree_guest.py\",\"start_line\":532,\"end_line\":544}]"}

env/patcheval/datasets/cve-2020-26215.jsonl

@@ -0,0 +1 @@
+{"cve_id":"CVE-2020-26215","work_dir":"/workspace/notebook","problem_statement":"Fix CVE-2020-26215 in Jupyter Notebook. Trailing-slash redirects can interpret repeated leading slashes as a protocol-relative external URL. Preserve the query string and normalize the redirect to a local path.\nVulnerability Location\n[{\"file_path\":\"notebook/base/handlers.py\",\"start_line\":861,\"end_line\":863}]"}

env/patcheval/datasets/cve-2021-23376.jsonl

@@ -0,0 +1 @@
+{"cve_id":"CVE-2021-23376","work_dir":"/workspace/ffmpegdotjs","problem_statement":"Fix CVE-2021-23376 in ffmpegdotjs. Attacker-controlled arguments to trimvideo reach child_process.exec and permit OS command injection. Preserve normal video trimming behavior.\nVulnerability Location\n[{\"file_path\":\"index.js\",\"start_line\":216,\"end_line\":233}]"}

env/patcheval/datasets/cve-2022-1986.jsonl

@@ -0,0 +1 @@
+{"cve_id":"CVE-2022-1986","work_dir":"/workspace/gogs","problem_statement":"Fix CVE-2022-1986 in Gogs. Repository path validation is vulnerable to OS command injection because Git path detection is platform-separator dependent. Handle both slash styles safely.\nVulnerability Location\n[{\"file_path\":\"internal/db/repo_editor.go\",\"start_line\":468,\"end_line\":474}]"}

env/patcheval/datasets/cve-2024-25620.jsonl

@@ -0,0 +1 @@
+{"cve_id":"CVE-2024-25620","work_dir":"/workspace/helm","problem_statement":"Fix CVE-2024-25620 in Helm. A chart name containing relative path changes can cause a chart to be saved outside the expected directory. Validate chart metadata names without breaking valid charts.\nVulnerability Location\n[{\"file_path\":\"pkg/chart/metadata.go\",\"start_line\":87,\"end_line\":146}]"}

env/patcheval/patcheval_config.yaml

@@ -0,0 +1,40 @@
+environments:
+  - env_name: patcheval_cve_2021_23376
+    env_image: ghcr.io/anonymous2578-data/cve-2021-23376:latest
+    env_num: 1
+    dataset: ./datasets/cve-2021-23376.jsonl
+    dataset_load_mode: eager
+    env_params:
+      task_family: patcheval
+
+  - env_name: patcheval_cve_2020_25459
+    env_image: ghcr.io/anonymous2578-data/cve-2020-25459:latest
+    env_num: 1
+    dataset: ./datasets/cve-2020-25459.jsonl
+    dataset_load_mode: eager
+    env_params:
+      task_family: patcheval
+
+  - env_name: patcheval_cve_2022_1986
+    env_image: ghcr.io/anonymous2578-data/cve-2022-1986:latest
+    env_num: 1
+    dataset: ./datasets/cve-2022-1986.jsonl
+    dataset_load_mode: eager
+    env_params:
+      task_family: patcheval
+
+  - env_name: patcheval_cve_2024_25620
+    env_image: ghcr.io/anonymous2578-data/cve-2024-25620:latest
+    env_num: 1
+    dataset: ./datasets/cve-2024-25620.jsonl
+    dataset_load_mode: eager
+    env_params:
+      task_family: patcheval
+
+  - env_name: patcheval_cve_2020_26215
+    env_image: ghcr.io/anonymous2578-data/cve-2020-26215:latest
+    env_num: 1
+    dataset: ./datasets/cve-2020-26215.jsonl
+    dataset_load_mode: eager
+    env_params:
+      task_family: patcheval

env/patcheval/patcheval_start.yaml

@@ -0,0 +1,21 @@
+agents:
+  patcheval_cve_2021_23376: &patcheval
+    container:
+      workdir: /workspace
+      runner_entrypoint:
+        source: ./runner.py
+        target: /tmp/safactory-patcheval-runner.py
+        command: "python /tmp/safactory-patcheval-runner.py"
+      install_runner_script: true
+      env:
+        PYTHONDONTWRITEBYTECODE: "1"
+        NO_PROXY: host.docker.internal,localhost,127.0.0.1,::1
+        no_proxy: host.docker.internal,localhost,127.0.0.1,::1
+      extra_args:
+        - --add-host=host.docker.internal:host-gateway
+      idle_command: "tail -f /dev/null"
+
+  patcheval_cve_2020_25459: *patcheval
+  patcheval_cve_2022_1986: *patcheval
+  patcheval_cve_2024_25620: *patcheval
+  patcheval_cve_2020_26215: *patcheval

env/patcheval/rule_evaluator.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import Any
+
+from evaluator.eval_types import EvalRequest, EvalResult, EvalSpec, EvalStatus, Trajectory
+
+
+async def evaluate_rule(
+    *,
+    request: EvalRequest,
+    spec: EvalSpec,
+    trajectory: Trajectory,
+) -> EvalResult:
+    """Commit the runner's staged PatchEval score as a 0-10 reward."""
+    metrics = _start_metrics(request)
+    score = _float_or_none(metrics.get("score"))
+    if score is None:
+        return EvalResult.failed(
+            session_id=request.session_id,
+            eval_id=spec.eval_id,
+            method=spec.method.value,
+            reason="patcheval metrics did not contain a numeric score",
+            artifacts={"bench": "patcheval", "metrics": metrics},
+        )
+
+    normalized_score = max(0.0, min(10.0, score))
+    return EvalResult(
+        session_id=request.session_id,
+        eval_id=spec.eval_id,
+        method=spec.method.value,
+        status=EvalStatus.SUCCEEDED.value,
+        raw_score=float(score),
+        normalized_score_10=round(normalized_score, 6),
+        reason="patcheval staged reward: apply=1, PoC=7, strict=10",
+        artifacts={
+            "bench": "patcheval",
+            "cve_id": metrics.get("cve_id"),
+            "strict_success": metrics.get("strict_success"),
+            "patch_applied": metrics.get("patch_applied"),
+            "poc_passed": metrics.get("poc_passed"),
+            "unit_tests_passed": metrics.get("unit_tests_passed"),
+            "failure_stage": metrics.get("failure_stage"),
+            "poc_log": metrics.get("poc_log"),
+            "unit_test_log": metrics.get("unit_test_log"),
+        },
+    )
+
+
+def _start_metrics(request: EvalRequest) -> dict[str, Any]:
+    start_result = getattr(request, "start_result", None)
+    metrics = getattr(start_result, "metrics", None)
+    return dict(metrics) if isinstance(metrics, dict) else {}
+
+
+def _float_or_none(value: Any) -> float | None:
+    try:
+        if value is None:
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None