Merge pull request #1019 from huggingface/add-pelican-svg-env

Add pelican_svg_env: blind SVG drawing, scored in three layers

Author: sergiopaniego

docs/source/_toctree.yml

@@ -129,6 +129,8 @@
     title: Agent World Model
   - local: environments/opencode
     title: OpenCode
+  - local: environments/pelican_svg
+    title: Pelican SVG
   - local: environments/sophistry_bench_sprint
     title: Sophistry Bench Sprint
   title: Environments

docs/source/environments.md

@@ -258,6 +258,14 @@ The OpenEnv community has built a catalog of ready-to-run environments that cove
         <a href="environments/opencode" class="!no-underline border dark:border-gray-700 px-3 py-1 rounded text-sm hover:shadow">📄 Docs</a>
       </div>
     </div>
+    <div class="border dark:border-gray-700 p-5 rounded-lg shadow">
+      <div class="font-bold mb-2">Pelican SVG</div>
+      <p class="text-sm"><code>pelican_svg_env</code> scores blind SVG drawings of an animal riding a vehicle in three layers: a source gate against cheats, deterministic geometry checks and a vision judge. 30 subject-vehicle tasks, with eval and GRPO training examples included.</p>
+      <div class="flex gap-2 mt-3">
+        <a href="environments/pelican_svg" class="!no-underline border dark:border-gray-700 px-3 py-1 rounded text-sm hover:shadow">📄 Docs</a>
+        <a href="https://huggingface.co/spaces/sergiopaniego/pelican-svg-env" class="!no-underline border dark:border-gray-700 px-3 py-1 rounded text-sm hover:shadow">🤗 HF</a>
+      </div>
+    </div>
     <div class="border dark:border-gray-700 p-5 rounded-lg shadow">
       <div class="font-bold mb-2">Sophistry Bench Sprint</div>
       <p class="text-sm"><code>sophistry_bench_sprint_env</code> is a single-turn advocacy reward-hacking environment on QuALITY passages: the policy defends an assigned answer and the reward proxy peaks at 8 <code>&lt;claim&gt;</code> tags, with four weight-0 canaries that detect format hacking.</p>

docs/source/environments/pelican_svg.md

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Pelican SVG environment for OpenEnv.
+
+Turns Simon Willison's "generate an SVG of a pelican riding a bicycle" check
+into an executable environment: the subject and vehicle are sampled from a
+grid so the canonical prompt is not the only thing being measured, and scoring
+runs in three layers of increasing cost.
+
+Examples:
+
+```python
+from envs.pelican_svg_env import PelicanSvgAction, PelicanSvgEnv
+
+with PelicanSvgEnv(base_url="http://localhost:8000") as env:
+    observation = env.reset().observation
+    result = env.step(PelicanSvgAction(response=my_model(observation.prompt)))
+    print(result.reward, result.observation.feedback)
+```
+"""
+
+from .client import PelicanSvgEnv
+from .models import PelicanSvgAction, PelicanSvgObservation, PelicanSvgState
+
+__all__ = [
+    "PelicanSvgEnv",
+    "PelicanSvgAction",
+    "PelicanSvgObservation",
+    "PelicanSvgState",
+]

envs/pelican_svg_env/README.md

@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Client for the Pelican SVG environment."""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from openenv.core.client_types import StepResult
+from openenv.core.env_client import EnvClient
+
+from .models import PelicanSvgAction, PelicanSvgObservation, PelicanSvgState
+
+
+class PelicanSvgEnv(
+    EnvClient[PelicanSvgAction, PelicanSvgObservation, PelicanSvgState]
+):
+    """Connects to a running Pelican SVG environment server.
+
+    Examples:
+
+    ```python
+    with PelicanSvgEnv(base_url="http://localhost:8000") as env:
+        observation = env.reset().observation
+        reply = my_model(observation.prompt)
+        result = env.step(PelicanSvgAction(response=reply))
+        print(result.reward, result.observation.feedback)
+    ```
+    """
+
+    def _step_payload(self, action: PelicanSvgAction) -> Dict[str, Any]:
+        """Convert an action into the JSON body of a step request."""
+        return {"response": action.response}
+
+    def _parse_result(
+        self, payload: Dict[str, Any]
+    ) -> StepResult[PelicanSvgObservation]:
+        """Parse a server response into a typed step result.
+
+        The server hoists `reward` and `done` onto the response envelope and
+        drops them from the serialised observation, so they are read from the
+        envelope first and only then from the observation body.
+        """
+        data = payload.get("observation", {})
+        reward = payload.get("reward", data.get("reward"))
+        done = payload.get("done", data.get("done", False))
+        observation = PelicanSvgObservation(
+            prompt=data.get("prompt", ""),
+            task_id=data.get("task_id", ""),
+            subject=data.get("subject", ""),
+            vehicle=data.get("vehicle", ""),
+            expected_wheels=data.get("expected_wheels", 2),
+            held_out=data.get("held_out", True),
+            feedback=data.get("feedback", ""),
+            gate_passed=data.get("gate_passed", False),
+            structure_score=data.get("structure_score", 0.0),
+            semantic_score=data.get("semantic_score", 0.0),
+            judged=data.get("judged", False),
+            violations=data.get("violations", []),
+            breakdown=data.get("breakdown", {}),
+            image_png_base64=data.get("image_png_base64"),
+            done=bool(done),
+            reward=reward,
+            metadata=payload.get("metadata", data.get("metadata", {})),
+        )
+        return StepResult(
+            observation=observation,
+            reward=observation.reward,
+            done=observation.done,
+        )
+
+    def _parse_state(self, payload: Dict[str, Any]) -> PelicanSvgState:
+        """Parse a response from the state endpoint into a typed state."""
+        return PelicanSvgState(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+            task_id=payload.get("task_id", ""),
+            submitted=payload.get("submitted", False),
+        )