Skip to content

Commit 0d6fa24

Browse files
abrichrclaude
andauthored
docs: add example Flask server for HttpAgent protocol (#120)
Minimal reference implementation showing the POST /act request/response contract. Copy-and-replace the predict() function with your model. Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent e8711c8 commit 0d6fa24

1 file changed

Lines changed: 134 additions & 0 deletions

File tree

examples/http_agent_server.py

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
"""Example Flask server implementing the HttpAgent protocol.
2+
3+
This is a minimal reference implementation showing the request/response
4+
contract for ``openadapt_evals.agents.HttpAgent``. Copy and adapt for
5+
your own model.
6+
7+
Usage:
8+
pip install flask pillow
9+
python examples/http_agent_server.py
10+
11+
# Then run eval against it:
12+
openadapt-evals run --agent http --agent-endpoint http://localhost:8080
13+
14+
Protocol:
15+
POST /act - Receive observation, return action
16+
POST /reset - (Optional) Reset agent state between episodes
17+
GET /health - Health check (return 200)
18+
"""
19+
20+
import base64
21+
import io
22+
import json
23+
import logging
24+
25+
from flask import Flask, jsonify, request
26+
27+
app = Flask(__name__)
28+
log = logging.getLogger(__name__)
29+
30+
# ---------------------------------------------------------------------------
31+
# Replace this with your model loading and inference logic
32+
# ---------------------------------------------------------------------------
33+
34+
35+
def load_model():
36+
"""Load your model here. Called once at startup."""
37+
log.info("Loading model... (replace this with your model loading code)")
38+
# Example:
39+
# from transformers import AutoModelForVision2Seq, AutoProcessor
40+
# model = AutoModelForVision2Seq.from_pretrained("your-model")
41+
# processor = AutoProcessor.from_pretrained("your-model")
42+
# return model, processor
43+
return None
44+
45+
46+
def predict(screenshot_bytes, instruction, viewport, step_count):
47+
"""Run your model on a screenshot and return an action dict.
48+
49+
Args:
50+
screenshot_bytes: PNG image bytes (or None).
51+
instruction: Task instruction string.
52+
viewport: [width, height] or None.
53+
step_count: How many steps have been taken so far.
54+
55+
Returns:
56+
Action dict, e.g.:
57+
{"type": "click", "x": 0.5, "y": 0.3}
58+
{"type": "type", "text": "hello world"}
59+
{"type": "key", "key": "Enter", "modifiers": ["ctrl"]}
60+
{"type": "scroll", "scroll_direction": "down"}
61+
{"type": "done"}
62+
63+
Coordinates should be in [0, 1] normalized range.
64+
"""
65+
# --- Replace everything below with your inference code ---
66+
log.info("Step %d: %s", step_count, instruction[:80])
67+
68+
# Dummy: always click center of screen
69+
return {"type": "click", "x": 0.5, "y": 0.5}
70+
71+
72+
# ---------------------------------------------------------------------------
73+
# HTTP endpoints (you probably don't need to modify these)
74+
# ---------------------------------------------------------------------------
75+
76+
MODEL = None
77+
78+
79+
@app.route("/act", methods=["POST"])
80+
def act():
81+
"""Receive observation, return action.
82+
83+
Request JSON:
84+
screenshot_b64: str | null - Base64-encoded PNG
85+
instruction: str - Task instruction
86+
task_id: str - Task identifier
87+
viewport: [int, int] | null - [width, height]
88+
accessibility_tree: dict | null
89+
step_count: int - Steps taken so far
90+
91+
Response JSON:
92+
type: str - "click", "type", "key", "scroll", "drag", "done"
93+
x: float | null - Normalized [0,1] x coordinate
94+
y: float | null - Normalized [0,1] y coordinate
95+
text: str | null - Text to type (for "type" action)
96+
key: str | null - Key name (for "key" action)
97+
modifiers: list | null - ["ctrl", "shift", "alt"]
98+
scroll_direction: str | null - "up", "down"
99+
target_node_id: str | null - A11y element ID
100+
"""
101+
data = request.get_json(force=True)
102+
103+
# Decode screenshot
104+
screenshot_bytes = None
105+
if data.get("screenshot_b64"):
106+
screenshot_bytes = base64.b64decode(data["screenshot_b64"])
107+
108+
action = predict(
109+
screenshot_bytes=screenshot_bytes,
110+
instruction=data.get("instruction", ""),
111+
viewport=data.get("viewport"),
112+
step_count=data.get("step_count", 0),
113+
)
114+
115+
return jsonify(action)
116+
117+
118+
@app.route("/reset", methods=["POST"])
119+
def reset():
120+
"""Optional: reset agent state between episodes."""
121+
log.info("Agent reset")
122+
return jsonify({"status": "ok"})
123+
124+
125+
@app.route("/health", methods=["GET"])
126+
def health():
127+
"""Health check."""
128+
return jsonify({"status": "ok"})
129+
130+
131+
if __name__ == "__main__":
132+
logging.basicConfig(level=logging.INFO)
133+
MODEL = load_model()
134+
app.run(host="0.0.0.0", port=8080)

0 commit comments

Comments
 (0)