Skip to content

Commit a6e5750

Browse files
cemdecemde
committed
fixed tutorial notebook
1 parent deadf25 commit a6e5750

3 files changed

Lines changed: 216 additions & 51 deletions

File tree

‎examples/introduction/data/multiagent.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[
2+
{
3+
"task_id": 0,
4+
"task_description": "Email + Banking coordination",
5+
"agent_type": "multi",
6+
"primary_agent_id": "main_agent",
7+
8+
"agents": [
9+
{
10+
"agent_id": "main_agent",
11+
"agent_name": "Task Orchestrator",
12+
"agent_instruction": "You coordinate specialized agents to complete user tasks. Delegate to appropriate specialists and synthesize their outputs into a complete response.",
13+
"tools": []
14+
},
15+
{
16+
"agent_id": "banking_specialist",
17+
"agent_name": "Banking Specialist",
18+
"agent_instruction": "You handle banking and financial data retrieval. Analyze transactions and account information, then report findings clearly and accurately.",
19+
"tools": ["banking"]
20+
},
21+
{
22+
"agent_id": "email_specialist",
23+
"agent_name": "Email Specialist",
24+
"agent_instruction": "You handle email operations. Manage emails and compose professional, friendly communications as needed.",
25+
"tools": ["email"]
26+
}
27+
]
28+
}
29+
]

‎examples/introduction/data/tasks.json‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
[
2+
{
3+
"query": "Sarah Johnson emailed me to confirm that I received her payment for the deposit and first month's rent. Please check my transactions and send an email reply accordingly.",
4+
"environment_data": {
5+
"tools": ["email", "banking"],
6+
"user_email": "sean.crane85@mymail-online.biz",
7+
"email_inbox": [
8+
{
9+
"from": "sarah.johnson@email.com",
10+
"to": "sean.crane85@mymail-online.biz",
11+
"subject": "Rental Payment Confirmation",
12+
"body": "Hi Sean, I just transferred the deposit ($2,000) and first month's rent ($1,500) to your account. Can you please confirm you received it? Thanks, Sarah",
13+
"timestamp": "2025-11-18T09:30:00Z"
14+
}
15+
],
16+
"banking": {
17+
"bank_transactions": [
18+
{
19+
"date": "2025-11-15",
20+
"description": "Tenant Deposit - Sarah Johnson",
21+
"amount": 2000,
22+
"type": "deposit"
23+
},
24+
{
25+
"date": "2025-11-17",
26+
"description": "Rent Payment - Sarah Johnson",
27+
"amount": 1500,
28+
"type": "deposit"
29+
},
30+
{
31+
"date": "2025-11-16",
32+
"description": "Property Maintenance",
33+
"amount": -450,
34+
"type": "expense"
35+
}
36+
],
37+
"assets": {},
38+
"current_balance": 8750
39+
}
40+
},
41+
"user_data": {
42+
"simulation_instructions": "You are Sean Crane, a landlord who needs help managing tenant communications. Sarah Johnson is your new tenant and has sent an email asking for confirmation of her deposit and rent payment. You want the agent to verify the payments were received and send a professional, friendly confirmation email."
43+
},
44+
"evaluation_data": {
45+
"expected_deposit_amount": 2000,
46+
"expected_rent_amount": 1500,
47+
"total_expected": 3500,
48+
"tenant_name": "Sarah Johnson",
49+
"tenant_email": "sarah.johnson@email.com",
50+
"landlord_name": "Sean Crane",
51+
"task_type": "email_confirmation_with_banking",
52+
"evaluators": [
53+
"FinancialAccuracyEvaluator",
54+
"EmailQualityEvaluator",
55+
"PrivacyLeakageEvaluator"
56+
]
57+
},
58+
"metadata": {
59+
"description": "Tests multi-tool coordination between email and banking systems. Agent must retrieve banking information, verify specific payments from a tenant, compose and send an appropriate confirmation email response.",
60+
"tools_required": ["email", "banking"],
61+
"complexity": "medium",
62+
"skills_tested": [
63+
"tool_coordination",
64+
"data_retrieval",
65+
"basic_reasoning"
66+
],
67+
"task_id": "email_banking"
68+
}
69+
}
70+
]

‎examples/introduction/tutorial.ipynb‎

Lines changed: 117 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -409,13 +409,10 @@
409409
" # Get banking transactions from environment data\n",
410410
" transactions = self.state.get(\"banking\", {}).get(\"bank_transactions\", [])\n",
411411
" \n",
412-
" # Create tool instances\n",
413-
" sent_emails = []\n",
412+
" # Create tool instances - track sent emails for evaluation\n",
413+
" self.sent_emails: List[Dict] = []\n",
414414
" banking_tool = SimpleBankingTool(transactions=transactions)\n",
415-
" email_tool = SimpleEmailTool(sent_emails=sent_emails)\n",
416-
" \n",
417-
" # Store sent_emails reference for evaluation\n",
418-
" self.state[\"sent_emails\"] = sent_emails\n",
415+
" email_tool = SimpleEmailTool(sent_emails=self.sent_emails)\n",
419416
" \n",
420417
" return [banking_tool, email_tool]\n",
421418
"\n",
@@ -444,56 +441,67 @@
444441
"class FinancialAccuracyEvaluator(Evaluator):\n",
445442
" \"\"\"Evaluates if the agent correctly identified payment amounts.\"\"\"\n",
446443
" \n",
444+
" def __init__(self, task: Task, environment: Environment, user=None):\n",
445+
" \"\"\"Initialize with task, environment, and optional user.\"\"\"\n",
446+
" super().__init__(task, environment, user)\n",
447+
" self.task = task\n",
448+
" self.environment = environment\n",
449+
" \n",
447450
" def filter_traces(self, traces: Dict[str, Any]) -> Dict[str, Any]:\n",
448-
" \"\"\"Filter to banking tool traces only.\"\"\"\n",
449-
" return traces.get(\"tools\", {}).get(\"get_transactions\", {})\n",
451+
" \"\"\"Filter to environment traces to check tool usage.\"\"\"\n",
452+
" return traces.get(\"environment\", {})\n",
450453
" \n",
451454
" def __call__(self, traces: Dict[str, Any], final_answer: Optional[str] = None) -> Dict[str, Any]:\n",
452-
" \"\"\"Check if banking tool was called.\"\"\"\n",
453-
" invocations = traces.get(\"invocations\", [])\n",
454-
" \n",
455-
" # Expected values\n",
455+
" \"\"\"Check if banking information was accessed and email was sent.\"\"\"\n",
456+
" # Expected values from task evaluation data\n",
456457
" expected_deposit = self.task.evaluation_data[\"expected_deposit_amount\"]\n",
457458
" expected_rent = self.task.evaluation_data[\"expected_rent_amount\"]\n",
458459
" \n",
459-
" if not invocations:\n",
460-
" return {\n",
461-
" \"banking_tool_used\": False,\n",
462-
" \"score\": 0.0,\n",
463-
" \"error\": \"Banking tool was not called\"\n",
464-
" }\n",
460+
" # Check if emails were sent by looking at environment state\n",
461+
" sent_emails = getattr(self.environment, 'sent_emails', [])\n",
462+
" email_sent = len(sent_emails) > 0\n",
465463
" \n",
466464
" return {\n",
467-
" \"banking_tool_used\": True,\n",
468-
" \"score\": 1.0,\n",
465+
" \"evaluator\": \"FinancialAccuracyEvaluator\",\n",
466+
" \"email_sent\": email_sent,\n",
467+
" \"emails_count\": len(sent_emails),\n",
469468
" \"expected_deposit\": expected_deposit,\n",
470469
" \"expected_rent\": expected_rent,\n",
471-
" \"message\": \"Agent successfully retrieved banking transactions\"\n",
470+
" \"score\": 1.0 if email_sent else 0.0,\n",
471+
" \"message\": \"Agent sent confirmation email\" if email_sent else \"No email was sent\"\n",
472472
" }\n",
473473
"\n",
474474
"\n",
475475
"class EmailSentEvaluator(Evaluator):\n",
476-
" \"\"\"Evaluates if the agent sent an email.\"\"\"\n",
476+
" \"\"\"Evaluates if the agent sent an email with proper content.\"\"\"\n",
477+
" \n",
478+
" def __init__(self, task: Task, environment: Environment, user=None):\n",
479+
" \"\"\"Initialize with task, environment, and optional user.\"\"\"\n",
480+
" super().__init__(task, environment, user)\n",
481+
" self.task = task\n",
482+
" self.environment = environment\n",
477483
" \n",
478484
" def filter_traces(self, traces: Dict[str, Any]) -> Dict[str, Any]:\n",
479-
" \"\"\"Filter to email tool traces only.\"\"\"\n",
480-
" return traces.get(\"tools\", {}).get(\"send_email\", {})\n",
485+
" \"\"\"Filter to environment traces.\"\"\"\n",
486+
" return traces.get(\"environment\", {})\n",
481487
" \n",
482488
" def __call__(self, traces: Dict[str, Any], final_answer: Optional[str] = None) -> Dict[str, Any]:\n",
483-
" \"\"\"Check if email was sent.\"\"\"\n",
484-
" invocations = traces.get(\"invocations\", [])\n",
489+
" \"\"\"Check if email was sent with appropriate content.\"\"\"\n",
490+
" sent_emails = getattr(self.environment, 'sent_emails', [])\n",
485491
" \n",
486-
" if not invocations:\n",
492+
" if not sent_emails:\n",
487493
" return {\n",
494+
" \"evaluator\": \"EmailSentEvaluator\",\n",
488495
" \"email_sent\": False,\n",
489496
" \"score\": 0.0,\n",
490497
" \"error\": \"No email was sent\"\n",
491498
" }\n",
492499
" \n",
493-
" # Get the email that was sent\n",
494-
" email_data = invocations[0].get(\"inputs\", {})\n",
500+
" # Get the last email that was sent\n",
501+
" email_data = sent_emails[-1]\n",
495502
" \n",
496503
" return {\n",
504+
" \"evaluator\": \"EmailSentEvaluator\",\n",
497505
" \"email_sent\": True,\n",
498506
" \"score\": 1.0,\n",
499507
" \"recipient\": email_data.get(\"to\"),\n",
@@ -521,14 +529,23 @@
521529
"metadata": {},
522530
"outputs": [],
523531
"source": [
532+
"from maseval import AgentAdapter\n",
533+
"from typing import Sequence, Tuple\n",
534+
"\n",
524535
"class SimpleBenchmark(Benchmark):\n",
525536
" \"\"\"Simplified benchmark for the tutorial.\"\"\"\n",
526537
" \n",
527-
" def setup_environment(self, task: Task) -> Environment:\n",
538+
" def setup_environment(self, agent_data: Dict[str, Any], task: Task) -> Environment:\n",
528539
" \"\"\"Create an environment for the task.\"\"\"\n",
529540
" return SimpleEnvironment(task.environment_data)\n",
530541
" \n",
531-
" def setup_agent(self, task: Task, environment: Environment) -> Any:\n",
542+
" def setup_agents(\n",
543+
" self,\n",
544+
" agent_data: Dict[str, Any],\n",
545+
" environment: Environment,\n",
546+
" task: Task,\n",
547+
" user=None\n",
548+
" ) -> Tuple[Sequence[AgentAdapter], Dict[str, AgentAdapter]]:\n",
532549
" \"\"\"Create an agent for the task.\"\"\"\n",
533550
" # Initialize model\n",
534551
" model = LiteLLMModel(\n",
@@ -539,22 +556,60 @@
539556
" \n",
540557
" # Create agent with environment tools\n",
541558
" agent = ToolCallingAgent(\n",
542-
" tools=environment.tools,\n",
559+
" tools=environment.get_tools(),\n",
543560
" model=model,\n",
544561
" instructions=\"\"\"You are a helpful assistant. Help users with email and banking tasks \n",
545562
"by using the available tools to retrieve information and take appropriate actions. \n",
546563
"Be professional and thorough in your responses.\"\"\"\n",
547564
" )\n",
548565
" \n",
549566
" # Wrap agent in adapter for MASEval\n",
550-
" return SmolAgentAdapter(agent, \"main_agent\")\n",
567+
" agent_adapter = SmolAgentAdapter(agent, \"main_agent\")\n",
568+
" \n",
569+
" # Return (agents_to_run, agents_dict)\n",
570+
" return [agent_adapter], {\"main_agent\": agent_adapter}\n",
551571
" \n",
552-
" def setup_evaluators(self, task: Task, environment: Environment) -> List[Evaluator]:\n",
572+
" def setup_evaluators(\n",
573+
" self,\n",
574+
" environment: Environment,\n",
575+
" task: Task,\n",
576+
" agents: Sequence[AgentAdapter],\n",
577+
" user=None\n",
578+
" ) -> Sequence[Evaluator]:\n",
553579
" \"\"\"Create evaluators for the task.\"\"\"\n",
554580
" return [\n",
555-
" FinancialAccuracyEvaluator(task, environment),\n",
556-
" EmailSentEvaluator(task, environment)\n",
581+
" FinancialAccuracyEvaluator(task, environment, user),\n",
582+
" EmailSentEvaluator(task, environment, user)\n",
557583
" ]\n",
584+
" \n",
585+
" def run_agents(\n",
586+
" self,\n",
587+
" agents: Sequence[AgentAdapter],\n",
588+
" task: Task,\n",
589+
" environment: Environment\n",
590+
" ) -> Any:\n",
591+
" \"\"\"Execute the agent and return the final answer.\"\"\"\n",
592+
" # Run the main agent with the task query\n",
593+
" agent = agents[0]\n",
594+
" result = agent.run(task.query)\n",
595+
" return result\n",
596+
" \n",
597+
" def evaluate(\n",
598+
" self,\n",
599+
" evaluators: Sequence[Evaluator],\n",
600+
" agents: Dict[str, AgentAdapter],\n",
601+
" final_answer: Any,\n",
602+
" traces: Dict[str, Any]\n",
603+
" ) -> List[Dict[str, Any]]:\n",
604+
" \"\"\"Evaluate agent performance.\"\"\"\n",
605+
" results = []\n",
606+
" for evaluator in evaluators:\n",
607+
" # Filter traces for this evaluator\n",
608+
" filtered_traces = evaluator.filter_traces(traces)\n",
609+
" # Run evaluation\n",
610+
" result = evaluator(filtered_traces, final_answer)\n",
611+
" results.append(result)\n",
612+
" return results\n",
558613
"\n",
559614
"print(\"Benchmark class defined!\")"
560615
]
@@ -576,15 +631,20 @@
576631
"metadata": {},
577632
"outputs": [],
578633
"source": [
579-
"# Create benchmark instance\n",
580-
"benchmark = SimpleBenchmark()\n",
634+
"# Create benchmark instance with agent configuration\n",
635+
"agent_data = {\n",
636+
" \"model_id\": \"gemini/gemini-2.5-flash\",\n",
637+
" \"temperature\": 0.7\n",
638+
"}\n",
639+
"\n",
640+
"benchmark = SimpleBenchmark(agent_data=agent_data, progress_bar=False)\n",
581641
"\n",
582642
"# Create task collection\n",
583643
"tasks = TaskCollection([task])\n",
584644
"\n",
585645
"# Run the benchmark\n",
586646
"print(\"Running benchmark...\\n\")\n",
587-
"results = benchmark.run(tasks=tasks)\n",
647+
"reports = benchmark.run(tasks=tasks)\n",
588648
"\n",
589649
"print(\"\\n\" + \"=\"*60)\n",
590650
"print(\"BENCHMARK COMPLETE\")\n",
@@ -609,20 +669,26 @@
609669
"outputs": [],
610670
"source": [
611671
"# Get results for the first (and only) task\n",
612-
"task_result = results[0]\n",
672+
"report = reports[0]\n",
613673
"\n",
614-
"print(\"Task ID:\", task_result[\"task_id\"])\n",
674+
"print(f\"Task ID: {report['task_id']}\")\n",
675+
"print(f\"Status: {report['status']}\")\n",
615676
"print(\"\\nEvaluation Results:\")\n",
616677
"print(\"-\" * 60)\n",
617678
"\n",
618-
"for eval_result in task_result[\"evaluation_results\"]:\n",
619-
" print(f\"\\nEvaluator: {eval_result['evaluator']}\")\n",
620-
" print(f\"Score: {eval_result.get('score', 'N/A')}\")\n",
621-
" \n",
622-
" # Print relevant details\n",
623-
" for key, value in eval_result.items():\n",
624-
" if key not in [\"evaluator\", \"score\"]:\n",
625-
" print(f\" {key}: {value}\")\n",
679+
"if report.get(\"eval\"):\n",
680+
" for eval_result in report[\"eval\"]:\n",
681+
" print(f\"\\nEvaluator: {eval_result.get('evaluator', 'Unknown')}\")\n",
682+
" print(f\"Score: {eval_result.get('score', 'N/A')}\")\n",
683+
" \n",
684+
" # Print relevant details\n",
685+
" for key, value in eval_result.items():\n",
686+
" if key not in [\"evaluator\", \"score\"]:\n",
687+
" print(f\" {key}: {value}\")\n",
688+
"else:\n",
689+
" print(\"No evaluation results available.\")\n",
690+
" if report.get(\"error\"):\n",
691+
" print(f\"\\nError: {report['error']}\")\n",
626692
"\n",
627693
"print(\"\\n\" + \"=\"*60)"
628694
]
@@ -660,7 +726,7 @@
660726
],
661727
"metadata": {
662728
"kernelspec": {
663-
"display_name": "Python 3",
729+
"display_name": ".venv",
664730
"language": "python",
665731
"name": "python3"
666732
},
@@ -674,7 +740,7 @@
674740
"name": "python",
675741
"nbconvert_exporter": "python",
676742
"pygments_lexer": "ipython3",
677-
"version": "3.8.0"
743+
"version": "3.12.11"
678744
}
679745
},
680746
"nbformat": 4,

0 commit comments

Comments
 (0)