|
| 1 | +import asyncio |
| 2 | +import os |
| 3 | +import signal |
| 4 | +import subprocess |
| 5 | +import sys |
| 6 | +import time |
| 7 | +import httpx |
| 8 | +import json |
| 9 | +import uuid |
| 10 | +from datetime import datetime |
| 11 | + |
| 12 | +# Configuration |
| 13 | +GREEN_AGENT_DIR = os.path.abspath("code_translator_green_agent") |
| 14 | +PURPLE_AGENT_DIR = os.path.abspath("code_translator_purple_agent") |
| 15 | +LEADERBOARD_DIR = os.path.abspath("code_translator_leaderboard") |
| 16 | +RESULTS_DIR = os.path.join(LEADERBOARD_DIR, "results") |
| 17 | + |
| 18 | +# Ensure results directory exists |
| 19 | +os.makedirs(RESULTS_DIR, exist_ok=True) |
| 20 | + |
| 21 | +GREEN_PORT = 9009 |
| 22 | +PURPLE_PORT = 9010 |
| 23 | + |
| 24 | +GREEN_URL = f"http://127.0.0.1:{GREEN_PORT}" |
| 25 | +PURPLE_URL = f"http://127.0.0.1:{PURPLE_PORT}" |
| 26 | + |
| 27 | +def start_process(command, cwd, name, log_file): |
| 28 | + print(f"Starting {name}...") |
| 29 | + f = open(log_file, "w") |
| 30 | + process = subprocess.Popen( |
| 31 | + command, |
| 32 | + cwd=cwd, |
| 33 | + stdout=f, |
| 34 | + stderr=subprocess.STDOUT, |
| 35 | + preexec_fn=os.setsid |
| 36 | + ) |
| 37 | + return process, f |
| 38 | + |
| 39 | +async def wait_for_agent(url, name, timeout=30): |
| 40 | + print(f"Waiting for {name} to be ready at {url}...") |
| 41 | + start_time = time.time() |
| 42 | + async with httpx.AsyncClient() as client: |
| 43 | + while time.time() - start_time < timeout: |
| 44 | + try: |
| 45 | + response = await client.get(f"{url}/.well-known/agent-card.json") |
| 46 | + if response.status_code == 200: |
| 47 | + print(f"{name} is ready!") |
| 48 | + return True |
| 49 | + except httpx.ConnectError: |
| 50 | + pass |
| 51 | + except Exception as e: |
| 52 | + print(f"Error checking {name}: {e}") |
| 53 | + await asyncio.sleep(1) |
| 54 | + print(f"{name} failed to start within {timeout} seconds.") |
| 55 | + return False |
| 56 | + |
| 57 | +async def run_leaderboard_test(): |
| 58 | + # 1. Start Agents |
| 59 | + green_proc, green_log = start_process( |
| 60 | + ["uv", "run", "src/server.py", "--port", str(GREEN_PORT)], |
| 61 | + GREEN_AGENT_DIR, |
| 62 | + "Green Agent", |
| 63 | + "green_agent_leaderboard.log" |
| 64 | + ) |
| 65 | + |
| 66 | + purple_proc, purple_log = start_process( |
| 67 | + ["uv", "run", "src/server.py", "--port", str(PURPLE_PORT)], |
| 68 | + PURPLE_AGENT_DIR, |
| 69 | + "Purple Agent", |
| 70 | + "purple_agent_leaderboard.log" |
| 71 | + ) |
| 72 | + |
| 73 | + try: |
| 74 | + # 2. Wait for health |
| 75 | + green_ready = await wait_for_agent(GREEN_URL, "Green Agent") |
| 76 | + purple_ready = await wait_for_agent(PURPLE_URL, "Purple Agent") |
| 77 | + |
| 78 | + if not (green_ready and purple_ready): |
| 79 | + print("One or more agents failed to start. Aborting.") |
| 80 | + return |
| 81 | + |
| 82 | + # 3. Send Evaluation Request |
| 83 | + print("\n--- Sending Evaluation Request for Leaderboard ---") |
| 84 | + |
| 85 | + participant_id = "019b8933-d5b6-76a3-8e0b-930c19c10e87" # Example ID from scenario |
| 86 | + participant_name = "translator" |
| 87 | + |
| 88 | + payload = { |
| 89 | + "participants": { |
| 90 | + participant_name: PURPLE_URL |
| 91 | + }, |
| 92 | + "config": { |
| 93 | + "test_cases": [ |
| 94 | + """ |
| 95 | +def process_data(items): |
| 96 | + \"\"\" |
| 97 | + Filters items with length > 3 and converts them to uppercase. |
| 98 | + Returns a list of formatted strings with their original indices. |
| 99 | + \"\"\" |
| 100 | + return [ |
| 101 | + f"{idx}: {item.upper()}" |
| 102 | + for idx, item in enumerate(items) |
| 103 | + if len(item) > 3 |
| 104 | + ] |
| 105 | +""", |
| 106 | + """ |
| 107 | +class Fibonacci: |
| 108 | + def __init__(self): |
| 109 | + self.memo = {} |
| 110 | +
|
| 111 | + def get_number(self, n): |
| 112 | + if n in self.memo: |
| 113 | + return self.memo[n] |
| 114 | + if n <= 1: |
| 115 | + return n |
| 116 | + self.memo[n] = self.get_number(n - 1) + self.get_number(n - 2) |
| 117 | + return self.memo[n] |
| 118 | +""", |
| 119 | + """ |
| 120 | +import re |
| 121 | +
|
| 122 | +def parse_log_line(line): |
| 123 | + # Format: [TIMESTAMP] LEVEL: Message |
| 124 | + pattern = r"\[(.*?)\] (\w+): (.*)" |
| 125 | + match = re.match(pattern, line) |
| 126 | + if match: |
| 127 | + timestamp, level, message = match.groups() |
| 128 | + return {"time": timestamp, "level": level, "msg": message} |
| 129 | + return None |
| 130 | +""" |
| 131 | + ], |
| 132 | + "source_language": "python", |
| 133 | + "target_language": "javascript" |
| 134 | + } |
| 135 | + } |
| 136 | + |
| 137 | + from a2a.client import A2ACardResolver, ClientConfig, ClientFactory |
| 138 | + from a2a.types import Message, Part, TextPart, Role, DataPart |
| 139 | + from uuid import uuid4 |
| 140 | + |
| 141 | + async with httpx.AsyncClient(timeout=120.0) as httpx_client: |
| 142 | + print(f"Resolving agent card from {GREEN_URL}...") |
| 143 | + resolver = A2ACardResolver(httpx_client=httpx_client, base_url=GREEN_URL) |
| 144 | + agent_card = await resolver.get_agent_card() |
| 145 | + |
| 146 | + config = ClientConfig(httpx_client=httpx_client, streaming=True) |
| 147 | + factory = ClientFactory(config) |
| 148 | + client = factory.create(agent_card) |
| 149 | + |
| 150 | + msg = Message( |
| 151 | + kind="message", |
| 152 | + role=Role.user, |
| 153 | + parts=[Part(TextPart(text=json.dumps(payload)))], |
| 154 | + message_id=uuid4().hex, |
| 155 | + ) |
| 156 | + |
| 157 | + print(f"Messaging Green Agent at {GREEN_URL}...") |
| 158 | + |
| 159 | + evaluation_result = None |
| 160 | + |
| 161 | + async for event in client.send_message(msg): |
| 162 | + if isinstance(event, Message): |
| 163 | + pass |
| 164 | + else: |
| 165 | + task, update = event |
| 166 | + # Check for artifacts where evaluation result is stored |
| 167 | + if task.artifacts: |
| 168 | + for artifact in task.artifacts: |
| 169 | + if artifact.name == "Evaluation Result": |
| 170 | + # Assuming it's a DataPart |
| 171 | + if isinstance(artifact.parts[0].root, DataPart): |
| 172 | + evaluation_result = artifact.parts[0].root.data |
| 173 | + print(f"[ARTIFACT] Found evaluation result: {evaluation_result}") |
| 174 | + |
| 175 | + if evaluation_result: |
| 176 | + # 4. Generate Leaderboard JSON |
| 177 | + print("\n--- Generating Leaderboard Entry ---") |
| 178 | + |
| 179 | + # Extract scores directly from the new schema |
| 180 | + execution_correctness = evaluation_result.get("execution_correctness", 0) |
| 181 | + style_score = evaluation_result.get("style_score", 0) |
| 182 | + conciseness = evaluation_result.get("conciseness", 0) |
| 183 | + relevance = evaluation_result.get("relevance", 0) |
| 184 | + |
| 185 | + # Calculate overall score if needed, or use average |
| 186 | + overall_score = (execution_correctness + style_score + conciseness + relevance) / 4.0 |
| 187 | + |
| 188 | + # Format matching AgentBeats tutorial (participants + results) |
| 189 | + result_entry = { |
| 190 | + "participants": { |
| 191 | + participant_name: participant_id |
| 192 | + }, |
| 193 | + "results": [ |
| 194 | + { |
| 195 | + "winner": evaluation_result.get("winner", participant_name), |
| 196 | + "execution_correctness": execution_correctness, |
| 197 | + "style_score": style_score, |
| 198 | + "conciseness": conciseness, |
| 199 | + "relevance": relevance, |
| 200 | + "overall_score": overall_score, |
| 201 | + "reasoning": evaluation_result.get("reasoning", "") |
| 202 | + } |
| 203 | + ] |
| 204 | + } |
| 205 | + |
| 206 | + filename = f"result_{participant_name}_{int(time.time())}.json" |
| 207 | + filepath = os.path.join(RESULTS_DIR, filename) |
| 208 | + |
| 209 | + with open(filepath, "w") as f: |
| 210 | + json.dump(result_entry, f, indent=2) |
| 211 | + |
| 212 | + print(f"Leaderboard result saved to: {filepath}") |
| 213 | + print(json.dumps(result_entry, indent=2)) |
| 214 | + |
| 215 | + else: |
| 216 | + print("Failed to retrieve evaluation result artifact.") |
| 217 | + |
| 218 | + except Exception as e: |
| 219 | + print(f"An error occurred during testing: {e}") |
| 220 | + |
| 221 | + finally: |
| 222 | + print("\n--- Shutting down agents ---") |
| 223 | + os.killpg(os.getpgid(green_proc.pid), signal.SIGTERM) |
| 224 | + os.killpg(os.getpgid(purple_proc.pid), signal.SIGTERM) |
| 225 | + green_log.close() |
| 226 | + purple_log.close() |
| 227 | + print("Done.") |
| 228 | + |
| 229 | +if __name__ == "__main__": |
| 230 | + asyncio.run(run_leaderboard_test()) |
0 commit comments