From 77a9e4d7d1e511587967cd5c41b10167e38998a7 Mon Sep 17 00:00:00 2001 From: frvade <1924461+frvade@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:39:21 +0000 Subject: [PATCH 1/3] feat: add ART MCP-RL taskset adapter --- verifiers/v1/tasksets/art_mcp/__init__.py | 19 +++++ verifiers/v1/tasksets/art_mcp/taskset.py | 97 +++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 verifiers/v1/tasksets/art_mcp/__init__.py create mode 100644 verifiers/v1/tasksets/art_mcp/taskset.py diff --git a/verifiers/v1/tasksets/art_mcp/__init__.py b/verifiers/v1/tasksets/art_mcp/__init__.py new file mode 100644 index 000000000..c9ad86d80 --- /dev/null +++ b/verifiers/v1/tasksets/art_mcp/__init__.py @@ -0,0 +1,19 @@ +"""ART MCP-RL scenario taskset adapter.""" + +from verifiers.v1.tasksets.art_mcp.taskset import ( + ArtMCPTask, + ArtMCPTaskData, + ArtMCPTaskset, + ArtMCPTasksetConfig, + art_rows_from_tasks, + load_art_rows, +) + +__all__ = [ + "ArtMCPTask", + "ArtMCPTaskData", + "ArtMCPTaskset", + "ArtMCPTasksetConfig", + "art_rows_from_tasks", + "load_art_rows", +] diff --git a/verifiers/v1/tasksets/art_mcp/taskset.py b/verifiers/v1/tasksets/art_mcp/taskset.py new file mode 100644 index 000000000..3a2cda0cc --- /dev/null +++ b/verifiers/v1/tasksets/art_mcp/taskset.py @@ -0,0 +1,97 @@ +"""Taskset adapter for OpenPipe ART MCP-RL scenario files. + +ART MCP-RL stores generated scenarios as JSON/JSONL rows with a natural-language +``task`` and optional metadata such as ``difficulty``. This module maps those +rows into native verifiers v1 tasks while preserving enough structured data to +export them back to ART-style rows. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Iterable + +from pydantic import Field + +import verifiers.v1 as vf + + +class ArtMCPTaskData(vf.TaskData): + source_task: str + difficulty: int = 1 + art_metadata: dict[str, Any] = Field(default_factory=dict) + + +class ArtMCPTask(vf.Task[ArtMCPTaskData]): + @vf.stop + async def single_turn(self, trace: vf.Trace) -> bool: + return trace.num_turns >= 1 + + +class ArtMCPTasksetConfig(vf.TasksetConfig): + path: str + system_prompt: str | None = None + + +def _read_rows(path: Path) -> Iterable[dict[str, Any]]: + if path.suffix == ".jsonl": + for line in path.read_text().splitlines(): + if line.strip(): + yield json.loads(line) + return + + data = json.loads(path.read_text()) + if isinstance(data, list): + yield from data + return + if isinstance(data, dict): + rows = data.get("scenarios", data.get("tasks")) + if isinstance(rows, list): + yield from rows + return + raise ValueError(f"unsupported ART scenario file shape: {path}") + + +def load_art_rows(path: str | Path) -> list[dict[str, Any]]: + rows = list(_read_rows(Path(path))) + for idx, row in enumerate(rows): + task = row.get("task") + if not isinstance(task, str) or not task.strip(): + raise ValueError(f"row {idx} missing non-empty 'task'") + difficulty = row.get("difficulty", 1) + if not isinstance(difficulty, int) or difficulty < 1: + raise ValueError(f"row {idx} has invalid difficulty: {difficulty!r}") + return rows + + +def art_rows_from_tasks(tasks: Iterable[ArtMCPTask]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for task in tasks: + row = dict(task.data.art_metadata) + row["task"] = task.data.source_task + row["difficulty"] = task.data.difficulty + rows.append(row) + return rows + + +class ArtMCPTaskset(vf.Taskset[ArtMCPTask, ArtMCPTasksetConfig]): + def load(self) -> list[ArtMCPTask]: + return [ + ArtMCPTask( + ArtMCPTaskData( + idx=i, + prompt=row["task"], + system_prompt=self.config.system_prompt, + source_task=row["task"], + difficulty=row.get("difficulty", 1), + art_metadata={ + key: value + for key, value in row.items() + if key not in {"task", "difficulty"} + }, + ), + self.config.task, + ) + for i, row in enumerate(load_art_rows(self.config.path)) + ] From a1c92d0e23687826a6cff9d2550dbc575451aeea Mon Sep 17 00:00:00 2001 From: frvade <1924461+frvade@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:06:46 +0000 Subject: [PATCH 2/3] fix: validate ART scenario wrapper rows --- verifiers/v1/tasksets/art_mcp/taskset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/tasksets/art_mcp/taskset.py b/verifiers/v1/tasksets/art_mcp/taskset.py index 3a2cda0cc..9faf243f1 100644 --- a/verifiers/v1/tasksets/art_mcp/taskset.py +++ b/verifiers/v1/tasksets/art_mcp/taskset.py @@ -46,7 +46,9 @@ def _read_rows(path: Path) -> Iterable[dict[str, Any]]: yield from data return if isinstance(data, dict): - rows = data.get("scenarios", data.get("tasks")) + rows = data.get("scenarios") + if not isinstance(rows, list): + rows = data.get("tasks") if isinstance(rows, list): yield from rows return @@ -56,6 +58,8 @@ def _read_rows(path: Path) -> Iterable[dict[str, Any]]: def load_art_rows(path: str | Path) -> list[dict[str, Any]]: rows = list(_read_rows(Path(path))) for idx, row in enumerate(rows): + if not isinstance(row, dict): + raise ValueError(f"row {idx} must be a JSON object") task = row.get("task") if not isinstance(task, str) or not task.strip(): raise ValueError(f"row {idx} missing non-empty 'task'") From 9e6ade2da027ca9414c38c21aecd51e3d6363b0d Mon Sep 17 00:00:00 2001 From: frvade <1924461+frvade@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:15:19 +0000 Subject: [PATCH 3/3] fix: allow multi-turn ART MCP rollouts --- verifiers/v1/tasksets/art_mcp/taskset.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/verifiers/v1/tasksets/art_mcp/taskset.py b/verifiers/v1/tasksets/art_mcp/taskset.py index 9faf243f1..31bc97724 100644 --- a/verifiers/v1/tasksets/art_mcp/taskset.py +++ b/verifiers/v1/tasksets/art_mcp/taskset.py @@ -24,9 +24,7 @@ class ArtMCPTaskData(vf.TaskData): class ArtMCPTask(vf.Task[ArtMCPTaskData]): - @vf.stop - async def single_turn(self, trace: vf.Trace) -> bool: - return trace.num_turns >= 1 + pass class ArtMCPTasksetConfig(vf.TasksetConfig):