From de9f4edf33354381595fda749eb84f0a8f765cef Mon Sep 17 00:00:00 2001 From: "wuqingfu.528" Date: Mon, 11 Aug 2025 19:50:26 +0800 Subject: [PATCH 1/5] feat: parsing the tracing file into evaluation cases --- tests/test_evaluator.py | 397 +++++++++++++++++++++++++++++ veadk/evaluation/base_evaluator.py | 89 ++++++- 2 files changed, 484 insertions(+), 2 deletions(-) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index e66b0ec9..9113295c 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -59,6 +59,383 @@ ], } +TRACE_SET_DATA = [ + { + "name": "execute_tool get_city_weather", + "span_id": 5421848634094108689, + "trace_id": 115143748123782151752771111946932434777, + "start_time": 1754884672226444000, + "end_time": 1754884672226993000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "gcp.vertex.agent", + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "get_city_weather", + "gen_ai.tool.description": "Retrieves the weather information of a given city. the args must in English", + "gen_ai.tool.call.id": "call_6ow5622pvcouw3tpvr7rfqtl", + "gcp.vertex.agent.tool_call_args": "{\"city\": \"Xi'an\"}", + "gcp.vertex.agent.event_id": "afdc4159-10f1-4f94-aa91-1003a9b6f1cc", + "gcp.vertex.agent.tool_response": "{\"result\": \"Cool, 18\u00b0C\"}", + "gcp.vertex.agent.llm_request": "{}", + "gcp.vertex.agent.llm_response": "{}", + "tool.name": "get_city_weather", + "tool.description": "Retrieves the weather information of a given city. the args must in English", + "tool.parameters": "{\"city\": \"Xi'an\"}", + "input.value": "{\"city\": \"Xi'an\"}", + "input.mime_type": "application/json", + "output.value": "{\"id\":\"call_6ow5622pvcouw3tpvr7rfqtl\",\"name\":\"get_city_weather\",\"response\":{\"result\":\"Cool, 18\u00b0C\"}}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "parent_span_id": 7997784243558253239 + }, + { + "name": "call_llm", + "span_id": 7997784243558253239, + "trace_id": 115143748123782151752771111946932434777, + "start_time": 1754884660688448000, + "end_time": 1754884672227098000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "gcp.vertex.agent", + "gen_ai.request.model": "openai/doubao-seed-1.6-250615", + "gcp.vertex.agent.invocation_id": "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441", + "gcp.vertex.agent.session_id": "veadk_example_session", + "gcp.vertex.agent.event_id": "9e94cb99-9f78-47db-a79c-f5422354953e", + "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}]}", + "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_6ow5622pvcouw3tpvr7rfqtl\",\"args\":{\"city\":\"Xi'an\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":496,\"prompt_token_count\":487,\"total_token_count\":983}}", + "gen_ai.usage.input_tokens": 487, + "gen_ai.usage.output_tokens": 983, + "llm.provider": "google", + "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}]}", + "input.mime_type": "application/json", + "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.model_name": "openai/doubao-seed-1.6-250615", + "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.input_messages.0.message.role": "system", + "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.1.message.role": "user", + "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", + "llm.input_messages.1.message.contents.0.message_content.type": "text", + "output.value": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_6ow5622pvcouw3tpvr7rfqtl\",\"args\":{\"city\":\"Xi'an\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":496,\"prompt_token_count\":487,\"total_token_count\":983}}", + "output.mime_type": "application/json", + "llm.token_count.total": 983, + "llm.token_count.prompt": 487, + "llm.token_count.completion": 496, + "llm.output_messages.0.message.role": "model", + "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_city_weather", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "gen_ai.prompt.0.role": "user", + "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in Xi'an?\"}]", + "gen_ai.completion.0.role": "model", + "gen_ai.completion.0.content": "[{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}]", + "gen_ai.usage.prompt_tokens": 487, + "gen_ai.usage.completion_tokens": 496, + "gen_ai.usage.total_tokens": 983, + "gen_ai.usage.cache_read_input_tokens": 0, + "gen_ai.usage.cache_create_input_tokens": 0, + "openinference.span.kind": "LLM" + }, + "parent_span_id": 14844888006539887900 + }, + { + "name": "call_llm", + "span_id": 7789424022423491416, + "trace_id": 115143748123782151752771111946932434777, + "start_time": 1754884672227703000, + "end_time": 1754884676664423000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "gcp.vertex.agent", + "gen_ai.request.model": "openai/doubao-seed-1.6-250615", + "gcp.vertex.agent.invocation_id": "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441", + "gcp.vertex.agent.session_id": "veadk_example_session", + "gcp.vertex.agent.event_id": "c0929124-9be0-4f75-a6ba-f7a531c9ccb6", + "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}]}", + "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679}}", + "gen_ai.usage.input_tokens": 547, + "gen_ai.usage.output_tokens": 679, + "llm.provider": "google", + "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}]}", + "input.mime_type": "application/json", + "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.model_name": "openai/doubao-seed-1.6-250615", + "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.input_messages.0.message.role": "system", + "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.1.message.role": "user", + "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", + "llm.input_messages.1.message.contents.0.message_content.type": "text", + "llm.input_messages.2.message.role": "model", + "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.input_messages.3.message.role": "tool", + "llm.input_messages.3.message.name": "get_city_weather", + "llm.input_messages.3.message.content": "{\"result\": \"Cool, 18\u00b0C\"}", + "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679}}", + "output.mime_type": "application/json", + "llm.token_count.total": 679, + "llm.token_count.prompt": 547, + "llm.token_count.completion": 132, + "llm.output_messages.0.message.role": "model", + "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", + "llm.output_messages.0.message.contents.0.message_content.type": "text", + "gen_ai.prompt.0.role": "user", + "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in Xi'an?\"}]", + "gen_ai.completion.0.role": "model", + "gen_ai.completion.0.content": "[{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\\u00b0C.\"}]", + "gen_ai.usage.prompt_tokens": 547, + "gen_ai.usage.completion_tokens": 132, + "gen_ai.usage.total_tokens": 679, + "gen_ai.usage.cache_read_input_tokens": 0, + "gen_ai.usage.cache_create_input_tokens": 0, + "openinference.span.kind": "LLM" + }, + "parent_span_id": 14844888006539887900 + }, + { + "name": "agent_run [chat_robot]", + "span_id": 14844888006539887900, + "trace_id": 115143748123782151752771111946932434777, + "start_time": 1754884660688136000, + "end_time": 1754884676664691000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "veadk", + "gen_ai.request.model": "openai/doubao-seed-1.6-250615", + "gen_ai.response.model": "openai/doubao-seed-1.6-250615", + "gen_ai.request.type": "completion", + "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679},\"invocation_id\":\"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"c0929124-9be0-4f75-a6ba-f7a531c9ccb6\",\"timestamp\":1754884672.227546}", + "output.mime_type": "application/json", + "openinference.span.kind": "AGENT" + }, + "parent_span_id": 2943363177785645047 + }, + { + "name": "invocation [veadk_default_app]", + "span_id": 2943363177785645047, + "trace_id": 115143748123782151752771111946932434777, + "start_time": 1754884660687962000, + "end_time": 1754884676664833000, + "attributes": { + "input.value": "{\"user_id\": \"veadk_default_user\", \"session_id\": \"veadk_example_session\", \"new_message\": \"{\\\"parts\\\": [{\\\"video_metadata\\\": null, \\\"thought\\\": null, \\\"inline_data\\\": null, \\\"file_data\\\": null, \\\"thought_signature\\\": null, \\\"code_execution_result\\\": null, \\\"executable_code\\\": null, \\\"function_call\\\": null, \\\"function_response\\\": null, \\\"text\\\": \\\"How is the weather like in Xi'an?\\\"}], \\\"role\\\": \\\"user\\\"}\", \"run_config\": \"{\\\"speech_config\\\": null, \\\"response_modalities\\\": null, \\\"save_input_blobs_as_artifacts\\\": false, \\\"support_cfc\\\": false, \\\"streaming_mode\\\": \\\"StreamingMode.NONE\\\", \\\"output_audio_transcription\\\": null, \\\"input_audio_transcription\\\": null, \\\"realtime_input_config\\\": null, \\\"enable_affective_dialog\\\": null, \\\"proactivity\\\": null, \\\"max_llm_calls\\\": 500}\"}", + "input.mime_type": "application/json", + "user.id": "veadk_default_user", + "session.id": "veadk_example_session", + "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679},\"invocation_id\":\"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"c0929124-9be0-4f75-a6ba-f7a531c9ccb6\",\"timestamp\":1754884672.227546}", + "output.mime_type": "application/json", + "openinference.span.kind": "CHAIN" + }, + "parent_span_id": None + }, + { + "name": "execute_tool get_city_weather", + "span_id": 10315930087681041331, + "trace_id": 225149951093293230286974602372571218832, + "start_time": 1754884684740875000, + "end_time": 1754884684741386000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "gcp.vertex.agent", + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "get_city_weather", + "gen_ai.tool.description": "Retrieves the weather information of a given city. the args must in English", + "gen_ai.tool.call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", + "gcp.vertex.agent.tool_call_args": "{\"city\": \"Beijing\"}", + "gcp.vertex.agent.event_id": "e4e1b9cb-c147-4fb7-8cf6-dfecc0e0b9a0", + "gcp.vertex.agent.tool_response": "{\"result\": \"Sunny, 25\u00b0C\"}", + "gcp.vertex.agent.llm_request": "{}", + "gcp.vertex.agent.llm_response": "{}", + "tool.name": "get_city_weather", + "tool.description": "Retrieves the weather information of a given city. the args must in English", + "tool.parameters": "{\"city\": \"Beijing\"}", + "input.value": "{\"city\": \"Beijing\"}", + "input.mime_type": "application/json", + "output.value": "{\"id\":\"call_4x60qqjspcwxfu3jtsw1m3sh\",\"name\":\"get_city_weather\",\"response\":{\"result\":\"Sunny, 25\u00b0C\"}}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "parent_span_id": 5741673860775380214 + }, + { + "name": "call_llm", + "span_id": 5741673860775380214, + "trace_id": 225149951093293230286974602372571218832, + "start_time": 1754884677052310000, + "end_time": 1754884684741523000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "gcp.vertex.agent", + "gen_ai.request.model": "openai/doubao-seed-1.6-250615", + "gcp.vertex.agent.invocation_id": "e-b4defdb1-d023-47fa-9962-acd1e95ba1e1", + "gcp.vertex.agent.session_id": "veadk_example_session", + "gcp.vertex.agent.event_id": "262f18e2-3756-4633-95ee-5e76fe07b754", + "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}]}", + "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_4x60qqjspcwxfu3jtsw1m3sh\",\"args\":{\"city\":\"Beijing\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":316,\"prompt_token_count\":584,\"total_token_count\":900}}", + "gen_ai.usage.input_tokens": 584, + "gen_ai.usage.output_tokens": 900, + "llm.provider": "google", + "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}]}", + "input.mime_type": "application/json", + "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.model_name": "openai/doubao-seed-1.6-250615", + "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.input_messages.0.message.role": "system", + "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.1.message.role": "user", + "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", + "llm.input_messages.1.message.contents.0.message_content.type": "text", + "llm.input_messages.2.message.role": "model", + "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.input_messages.3.message.role": "tool", + "llm.input_messages.3.message.name": "get_city_weather", + "llm.input_messages.3.message.content": "{\"result\": \"Cool, 18\u00b0C\"}", + "llm.input_messages.4.message.role": "model", + "llm.input_messages.4.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", + "llm.input_messages.4.message.contents.0.message_content.type": "text", + "llm.input_messages.5.message.role": "user", + "llm.input_messages.5.message.contents.0.message_content.text": "How is the weather like in BeiJing?", + "llm.input_messages.5.message.contents.0.message_content.type": "text", + "output.value": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_4x60qqjspcwxfu3jtsw1m3sh\",\"args\":{\"city\":\"Beijing\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":316,\"prompt_token_count\":584,\"total_token_count\":900}}", + "output.mime_type": "application/json", + "llm.token_count.total": 900, + "llm.token_count.prompt": 584, + "llm.token_count.completion": 316, + "llm.output_messages.0.message.role": "model", + "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_city_weather", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Beijing\"}", + "gen_ai.prompt.0.role": "user", + "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in BeiJing?\"}]", + "gen_ai.completion.0.role": "model", + "gen_ai.completion.0.content": "[{\"function_call\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"args\": {\"city\": \"Beijing\"}, \"name\": \"get_city_weather\"}}]", + "gen_ai.usage.prompt_tokens": 584, + "gen_ai.usage.completion_tokens": 316, + "gen_ai.usage.total_tokens": 900, + "gen_ai.usage.cache_read_input_tokens": 0, + "gen_ai.usage.cache_create_input_tokens": 0, + "openinference.span.kind": "LLM" + }, + "parent_span_id": 8830792093920828159 + }, + { + "name": "call_llm", + "span_id": 1294773233164416528, + "trace_id": 225149951093293230286974602372571218832, + "start_time": 1754884684742479000, + "end_time": 1754884692281442000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "gcp.vertex.agent", + "gen_ai.request.model": "openai/doubao-seed-1.6-250615", + "gcp.vertex.agent.invocation_id": "e-b4defdb1-d023-47fa-9962-acd1e95ba1e1", + "gcp.vertex.agent.session_id": "veadk_example_session", + "gcp.vertex.agent.event_id": "2aad3f22-94da-4e55-9976-56b2256b61fc", + "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"args\": {\"city\": \"Beijing\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Sunny, 25\u00b0C\"}}}], \"role\": \"user\"}]}", + "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900}}", + "gen_ai.usage.input_tokens": 644, + "gen_ai.usage.output_tokens": 900, + "llm.provider": "google", + "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"args\": {\"city\": \"Beijing\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Sunny, 25\u00b0C\"}}}], \"role\": \"user\"}]}", + "input.mime_type": "application/json", + "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.model_name": "openai/doubao-seed-1.6-250615", + "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.input_messages.0.message.role": "system", + "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.1.message.role": "user", + "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", + "llm.input_messages.1.message.contents.0.message_content.type": "text", + "llm.input_messages.2.message.role": "model", + "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.input_messages.3.message.role": "tool", + "llm.input_messages.3.message.name": "get_city_weather", + "llm.input_messages.3.message.content": "{\"result\": \"Cool, 18\u00b0C\"}", + "llm.input_messages.4.message.role": "model", + "llm.input_messages.4.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", + "llm.input_messages.4.message.contents.0.message_content.type": "text", + "llm.input_messages.5.message.role": "user", + "llm.input_messages.5.message.contents.0.message_content.text": "How is the weather like in BeiJing?", + "llm.input_messages.5.message.contents.0.message_content.type": "text", + "llm.input_messages.6.message.role": "model", + "llm.input_messages.6.message.tool_calls.0.tool_call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", + "llm.input_messages.6.message.tool_calls.0.tool_call.function.name": "get_city_weather", + "llm.input_messages.6.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Beijing\"}", + "llm.input_messages.7.message.role": "tool", + "llm.input_messages.7.message.name": "get_city_weather", + "llm.input_messages.7.message.content": "{\"result\": \"Sunny, 25\u00b0C\"}", + "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900}}", + "output.mime_type": "application/json", + "llm.token_count.total": 900, + "llm.token_count.prompt": 644, + "llm.token_count.completion": 256, + "llm.output_messages.0.message.role": "model", + "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Beijing is sunny, with a temperature of 25\u00b0C.", + "llm.output_messages.0.message.contents.0.message_content.type": "text", + "gen_ai.prompt.0.role": "user", + "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in BeiJing?\"}]", + "gen_ai.completion.0.role": "model", + "gen_ai.completion.0.content": "[{\"text\": \"The weather in Beijing is sunny, with a temperature of 25\\u00b0C.\"}]", + "gen_ai.usage.prompt_tokens": 644, + "gen_ai.usage.completion_tokens": 256, + "gen_ai.usage.total_tokens": 900, + "gen_ai.usage.cache_read_input_tokens": 0, + "gen_ai.usage.cache_create_input_tokens": 0, + "openinference.span.kind": "LLM" + }, + "parent_span_id": 8830792093920828159 + }, + { + "name": "agent_run [chat_robot]", + "span_id": 8830792093920828159, + "trace_id": 225149951093293230286974602372571218832, + "start_time": 1754884677050889000, + "end_time": 1754884692281791000, + "attributes": { + "session.id": "veadk_example_session", + "user.id": "veadk_default_user", + "gen_ai.system": "veadk", + "gen_ai.request.model": "openai/doubao-seed-1.6-250615", + "gen_ai.response.model": "openai/doubao-seed-1.6-250615", + "gen_ai.request.type": "completion", + "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900},\"invocation_id\":\"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"2aad3f22-94da-4e55-9976-56b2256b61fc\",\"timestamp\":1754884684.74225}", + "output.mime_type": "application/json", + "openinference.span.kind": "AGENT" + }, + "parent_span_id": 6624741805060007734 + }, + { + "name": "invocation [veadk_default_app]", + "span_id": 6624741805060007734, + "trace_id": 225149951093293230286974602372571218832, + "start_time": 1754884677050051000, + "end_time": 1754884692281935000, + "attributes": { + "input.value": "{\"user_id\": \"veadk_default_user\", \"session_id\": \"veadk_example_session\", \"new_message\": \"{\\\"parts\\\": [{\\\"video_metadata\\\": null, \\\"thought\\\": null, \\\"inline_data\\\": null, \\\"file_data\\\": null, \\\"thought_signature\\\": null, \\\"code_execution_result\\\": null, \\\"executable_code\\\": null, \\\"function_call\\\": null, \\\"function_response\\\": null, \\\"text\\\": \\\"How is the weather like in BeiJing?\\\"}], \\\"role\\\": \\\"user\\\"}\", \"run_config\": \"{\\\"speech_config\\\": null, \\\"response_modalities\\\": null, \\\"save_input_blobs_as_artifacts\\\": false, \\\"support_cfc\\\": false, \\\"streaming_mode\\\": \\\"StreamingMode.NONE\\\", \\\"output_audio_transcription\\\": null, \\\"input_audio_transcription\\\": null, \\\"realtime_input_config\\\": null, \\\"enable_affective_dialog\\\": null, \\\"proactivity\\\": null, \\\"max_llm_calls\\\": 500}\"}", + "input.mime_type": "application/json", + "user.id": "veadk_default_user", + "session.id": "veadk_example_session", + "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900},\"invocation_id\":\"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"2aad3f22-94da-4e55-9976-56b2256b61fc\",\"timestamp\":1754884684.74225}", + "output.mime_type": "application/json", + "openinference.span.kind": "CHAIN" + }, + "parent_span_id": None + } +] + def test_evaluator(): base_evaluator = BaseEvaluator(agent=None, name="test_evaluator") @@ -78,3 +455,23 @@ def test_evaluator(): ) os.remove(eval_set_file_path) + + +def test_evaluator_using_tracing(): + base_evaluator = BaseEvaluator(agent=None, name="test_evaluator") + + # save data to file + tracing_set_file_path = "./tracing_set_for_test_evaluator.json" + with open(tracing_set_file_path, "w") as f: + json.dump(TRACE_SET_DATA, f) + + base_evaluator.generate_eval_data_from_tracing(tracing_set_file_path=tracing_set_file_path) + + assert len(base_evaluator.invocation_list) == 1 + assert len(base_evaluator.invocation_list[0].invocations) == 2 + assert ( + base_evaluator.invocation_list[0].invocations[0].invocation_id + == "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441" + ) + + os.remove(tracing_set_file_path) \ No newline at end of file diff --git a/veadk/evaluation/base_evaluator.py b/veadk/evaluation/base_evaluator.py index a5bf250c..18f359ac 100644 --- a/veadk/evaluation/base_evaluator.py +++ b/veadk/evaluation/base_evaluator.py @@ -13,6 +13,7 @@ # limitations under the License. +import json import time import uuid from abc import abstractmethod @@ -24,6 +25,8 @@ from google.genai import types from pydantic import BaseModel +from veadk.utils.misc import formatted_timestamp + class InvocationTestData(BaseModel): invocation_id: str = "" @@ -79,15 +82,97 @@ def __init__( self.result_list: list[EvalResultData] = [] self.agent_information_list: list[dict] = [] - def load_eval_set(self, eval_set_file: str) -> list[EvalSet]: + def load_eval_set(self, eval_set_file: str) -> EvalSet: from .eval_set_file_loader import load_eval_set_from_file return load_eval_set_from_file(eval_set_file) def generate_eval_data(self, eval_set_file_path: str): eval_case_data_list: list[EvalCaseData] = [] - eval_cases = self.load_eval_set(eval_set_file_path).eval_cases + self.generate_invocation_data(eval_cases, eval_case_data_list) + + def load_tracing_set(self, tracing_set_file_path: str) -> EvalSet: + with open(tracing_set_file_path, "r") as f: + tracing_data = json.load(f) + + # Group spans by trace_id + trace_groups = {} + for span in tracing_data: + trace_id = span["trace_id"] + if trace_id not in trace_groups: + trace_groups[trace_id] = [] + trace_groups[trace_id].append(span) + + # Convert to evalset format + eval_cases, conversation = [], [] + app_name, user_id = "", "" + creation_timestamp = 0 + for trace_id, spans in trace_groups.items(): + tool_uses = [] + + # Extract tool_uses from spans with name starting with "execute_tool" + for span in spans: + if span["name"].startswith("execute_tool"): + tool_uses.append({ + "id": span["attributes"].get("gen_ai.tool.call.id", None), + "args": json.loads(span["attributes"].get("gcp.vertex.agent.tool_call_args", "{}")), + "name": span["attributes"].get("gen_ai.tool.name", None), + }) + + # Extract conversation data from spans with name starting with "invocation" + for span in spans: + if span["name"].startswith("invocation"): + # Parse input.value and output.value as JSON + input_value = json.loads(span["attributes"].get("input.value", "{}")) + output_value = json.loads(span["attributes"].get("output.value", "{}")) + + user_content = json.loads(input_value.get("new_message", {})) + final_response = json.loads(json.dumps(user_content)) + final_response["parts"][0]["text"] = output_value.get("content", {}).get("parts", [{}])[0].get("text", None) + final_response["role"] = None + conversation.append({ + "invocation_id": output_value.get("invocation_id", str(uuid.uuid4())), + "user_content": user_content, + "final_response": final_response, + "intermediate_data": { + "tool_uses": tool_uses, + "intermediate_responses": [] + }, + "creation_timestamp": span["start_time"] / 1e9, + }) + user_id = input_value.get("user_id", None) + app_name = span["name"].replace("invocation", "").strip().strip("[]") + creation_timestamp = span["start_time"] / 1e9 + + + eval_cases.append({ + "eval_id": f"veadk_eval_{formatted_timestamp()}", + "conversation": conversation, + "session_input": { + "app_name": app_name, + "user_id": user_id, + "state": {}, + }, + "creation_timestamp": creation_timestamp, + }) + + evalset = EvalSet( + eval_set_id="default", + name="default", + description=None, + eval_cases=eval_cases, + creation_timestamp=creation_timestamp, + ) + + return evalset + + def generate_eval_data_from_tracing(self, tracing_set_file_path: str): + eval_case_data_list: list[EvalCaseData] = [] + eval_cases = self.load_tracing_set(tracing_set_file_path).eval_cases + self.generate_invocation_data(eval_cases, eval_case_data_list) + + def generate_invocation_data(self, eval_cases: list[EvalSet], eval_case_data_list: list[EvalCaseData]): for eval_case in eval_cases: eval_case_data = EvalCaseData(invocations=[]) self.agent_information_list.append( From d2cb21574a86c9981d2e2800fcc0bced417a3601 Mon Sep 17 00:00:00 2001 From: "wuqingfu.528" Date: Mon, 11 Aug 2025 19:59:48 +0800 Subject: [PATCH 2/5] feat: parsing the tracing file into evaluation cases --- tests/test_evaluator.py | 172 +++++++++++++++-------------- veadk/evaluation/base_evaluator.py | 89 +++++++++------ 2 files changed, 143 insertions(+), 118 deletions(-) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 9113295c..de1878f0 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -74,21 +74,21 @@ "gen_ai.tool.name": "get_city_weather", "gen_ai.tool.description": "Retrieves the weather information of a given city. the args must in English", "gen_ai.tool.call.id": "call_6ow5622pvcouw3tpvr7rfqtl", - "gcp.vertex.agent.tool_call_args": "{\"city\": \"Xi'an\"}", + "gcp.vertex.agent.tool_call_args": '{"city": "Xi\'an"}', "gcp.vertex.agent.event_id": "afdc4159-10f1-4f94-aa91-1003a9b6f1cc", - "gcp.vertex.agent.tool_response": "{\"result\": \"Cool, 18\u00b0C\"}", + "gcp.vertex.agent.tool_response": '{"result": "Cool, 18\u00b0C"}', "gcp.vertex.agent.llm_request": "{}", "gcp.vertex.agent.llm_response": "{}", "tool.name": "get_city_weather", "tool.description": "Retrieves the weather information of a given city. the args must in English", - "tool.parameters": "{\"city\": \"Xi'an\"}", - "input.value": "{\"city\": \"Xi'an\"}", + "tool.parameters": '{"city": "Xi\'an"}', + "input.value": '{"city": "Xi\'an"}', "input.mime_type": "application/json", - "output.value": "{\"id\":\"call_6ow5622pvcouw3tpvr7rfqtl\",\"name\":\"get_city_weather\",\"response\":{\"result\":\"Cool, 18\u00b0C\"}}", + "output.value": '{"id":"call_6ow5622pvcouw3tpvr7rfqtl","name":"get_city_weather","response":{"result":"Cool, 18\u00b0C"}}', "output.mime_type": "application/json", - "openinference.span.kind": "TOOL" + "openinference.span.kind": "TOOL", }, - "parent_span_id": 7997784243558253239 + "parent_span_id": 7997784243558253239, }, { "name": "call_llm", @@ -104,22 +104,22 @@ "gcp.vertex.agent.invocation_id": "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441", "gcp.vertex.agent.session_id": "veadk_example_session", "gcp.vertex.agent.event_id": "9e94cb99-9f78-47db-a79c-f5422354953e", - "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}]}", - "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_6ow5622pvcouw3tpvr7rfqtl\",\"args\":{\"city\":\"Xi'an\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":496,\"prompt_token_count\":487,\"total_token_count\":983}}", + "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}]}', + "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"function_call":{"id":"call_6ow5622pvcouw3tpvr7rfqtl","args":{"city":"Xi\'an"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":496,"prompt_token_count":487,"total_token_count":983}}', "gen_ai.usage.input_tokens": 487, "gen_ai.usage.output_tokens": 983, "llm.provider": "google", - "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}]}", + "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}]}', "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', "llm.input_messages.1.message.role": "user", "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", "llm.input_messages.1.message.contents.0.message_content.type": "text", - "output.value": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_6ow5622pvcouw3tpvr7rfqtl\",\"args\":{\"city\":\"Xi'an\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":496,\"prompt_token_count\":487,\"total_token_count\":983}}", + "output.value": '{"content":{"parts":[{"function_call":{"id":"call_6ow5622pvcouw3tpvr7rfqtl","args":{"city":"Xi\'an"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":496,"prompt_token_count":487,"total_token_count":983}}', "output.mime_type": "application/json", "llm.token_count.total": 983, "llm.token_count.prompt": 487, @@ -127,19 +127,19 @@ "llm.output_messages.0.message.role": "model", "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in Xi'an?\"}]", + "gen_ai.prompt.0.content": '[{"text": "How is the weather like in Xi\'an?"}]', "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": "[{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}]", + "gen_ai.completion.0.content": '[{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}]', "gen_ai.usage.prompt_tokens": 487, "gen_ai.usage.completion_tokens": 496, "gen_ai.usage.total_tokens": 983, "gen_ai.usage.cache_read_input_tokens": 0, "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM" + "openinference.span.kind": "LLM", }, - "parent_span_id": 14844888006539887900 + "parent_span_id": 14844888006539887900, }, { "name": "call_llm", @@ -155,29 +155,29 @@ "gcp.vertex.agent.invocation_id": "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441", "gcp.vertex.agent.session_id": "veadk_example_session", "gcp.vertex.agent.event_id": "c0929124-9be0-4f75-a6ba-f7a531c9ccb6", - "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}]}", - "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679}}", + "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}]}', + "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679}}', "gen_ai.usage.input_tokens": 547, "gen_ai.usage.output_tokens": 679, "llm.provider": "google", - "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}]}", + "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}]}', "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', "llm.input_messages.1.message.role": "user", "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", "llm.input_messages.1.message.contents.0.message_content.type": "text", "llm.input_messages.2.message.role": "model", "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', "llm.input_messages.3.message.role": "tool", "llm.input_messages.3.message.name": "get_city_weather", - "llm.input_messages.3.message.content": "{\"result\": \"Cool, 18\u00b0C\"}", - "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679}}", + "llm.input_messages.3.message.content": '{"result": "Cool, 18\u00b0C"}', + "output.value": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679}}', "output.mime_type": "application/json", "llm.token_count.total": 679, "llm.token_count.prompt": 547, @@ -186,17 +186,17 @@ "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", "llm.output_messages.0.message.contents.0.message_content.type": "text", "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in Xi'an?\"}]", + "gen_ai.prompt.0.content": '[{"text": "How is the weather like in Xi\'an?"}]', "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": "[{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\\u00b0C.\"}]", + "gen_ai.completion.0.content": '[{"text": "The weather in Xi\'an is cool, with a temperature of 18\\u00b0C."}]', "gen_ai.usage.prompt_tokens": 547, "gen_ai.usage.completion_tokens": 132, "gen_ai.usage.total_tokens": 679, "gen_ai.usage.cache_read_input_tokens": 0, "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM" + "openinference.span.kind": "LLM", }, - "parent_span_id": 14844888006539887900 + "parent_span_id": 14844888006539887900, }, { "name": "agent_run [chat_robot]", @@ -211,11 +211,11 @@ "gen_ai.request.model": "openai/doubao-seed-1.6-250615", "gen_ai.response.model": "openai/doubao-seed-1.6-250615", "gen_ai.request.type": "completion", - "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679},\"invocation_id\":\"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"c0929124-9be0-4f75-a6ba-f7a531c9ccb6\",\"timestamp\":1754884672.227546}", + "output.value": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679},"invocation_id":"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"c0929124-9be0-4f75-a6ba-f7a531c9ccb6","timestamp":1754884672.227546}', "output.mime_type": "application/json", - "openinference.span.kind": "AGENT" + "openinference.span.kind": "AGENT", }, - "parent_span_id": 2943363177785645047 + "parent_span_id": 2943363177785645047, }, { "name": "invocation [veadk_default_app]", @@ -224,15 +224,15 @@ "start_time": 1754884660687962000, "end_time": 1754884676664833000, "attributes": { - "input.value": "{\"user_id\": \"veadk_default_user\", \"session_id\": \"veadk_example_session\", \"new_message\": \"{\\\"parts\\\": [{\\\"video_metadata\\\": null, \\\"thought\\\": null, \\\"inline_data\\\": null, \\\"file_data\\\": null, \\\"thought_signature\\\": null, \\\"code_execution_result\\\": null, \\\"executable_code\\\": null, \\\"function_call\\\": null, \\\"function_response\\\": null, \\\"text\\\": \\\"How is the weather like in Xi'an?\\\"}], \\\"role\\\": \\\"user\\\"}\", \"run_config\": \"{\\\"speech_config\\\": null, \\\"response_modalities\\\": null, \\\"save_input_blobs_as_artifacts\\\": false, \\\"support_cfc\\\": false, \\\"streaming_mode\\\": \\\"StreamingMode.NONE\\\", \\\"output_audio_transcription\\\": null, \\\"input_audio_transcription\\\": null, \\\"realtime_input_config\\\": null, \\\"enable_affective_dialog\\\": null, \\\"proactivity\\\": null, \\\"max_llm_calls\\\": 500}\"}", + "input.value": '{"user_id": "veadk_default_user", "session_id": "veadk_example_session", "new_message": "{\\"parts\\": [{\\"video_metadata\\": null, \\"thought\\": null, \\"inline_data\\": null, \\"file_data\\": null, \\"thought_signature\\": null, \\"code_execution_result\\": null, \\"executable_code\\": null, \\"function_call\\": null, \\"function_response\\": null, \\"text\\": \\"How is the weather like in Xi\'an?\\"}], \\"role\\": \\"user\\"}", "run_config": "{\\"speech_config\\": null, \\"response_modalities\\": null, \\"save_input_blobs_as_artifacts\\": false, \\"support_cfc\\": false, \\"streaming_mode\\": \\"StreamingMode.NONE\\", \\"output_audio_transcription\\": null, \\"input_audio_transcription\\": null, \\"realtime_input_config\\": null, \\"enable_affective_dialog\\": null, \\"proactivity\\": null, \\"max_llm_calls\\": 500}"}', "input.mime_type": "application/json", "user.id": "veadk_default_user", "session.id": "veadk_example_session", - "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":132,\"prompt_token_count\":547,\"total_token_count\":679},\"invocation_id\":\"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"c0929124-9be0-4f75-a6ba-f7a531c9ccb6\",\"timestamp\":1754884672.227546}", + "output.value": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679},"invocation_id":"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"c0929124-9be0-4f75-a6ba-f7a531c9ccb6","timestamp":1754884672.227546}', "output.mime_type": "application/json", - "openinference.span.kind": "CHAIN" + "openinference.span.kind": "CHAIN", }, - "parent_span_id": None + "parent_span_id": None, }, { "name": "execute_tool get_city_weather", @@ -248,21 +248,21 @@ "gen_ai.tool.name": "get_city_weather", "gen_ai.tool.description": "Retrieves the weather information of a given city. the args must in English", "gen_ai.tool.call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", - "gcp.vertex.agent.tool_call_args": "{\"city\": \"Beijing\"}", + "gcp.vertex.agent.tool_call_args": '{"city": "Beijing"}', "gcp.vertex.agent.event_id": "e4e1b9cb-c147-4fb7-8cf6-dfecc0e0b9a0", - "gcp.vertex.agent.tool_response": "{\"result\": \"Sunny, 25\u00b0C\"}", + "gcp.vertex.agent.tool_response": '{"result": "Sunny, 25\u00b0C"}', "gcp.vertex.agent.llm_request": "{}", "gcp.vertex.agent.llm_response": "{}", "tool.name": "get_city_weather", "tool.description": "Retrieves the weather information of a given city. the args must in English", - "tool.parameters": "{\"city\": \"Beijing\"}", - "input.value": "{\"city\": \"Beijing\"}", + "tool.parameters": '{"city": "Beijing"}', + "input.value": '{"city": "Beijing"}', "input.mime_type": "application/json", - "output.value": "{\"id\":\"call_4x60qqjspcwxfu3jtsw1m3sh\",\"name\":\"get_city_weather\",\"response\":{\"result\":\"Sunny, 25\u00b0C\"}}", + "output.value": '{"id":"call_4x60qqjspcwxfu3jtsw1m3sh","name":"get_city_weather","response":{"result":"Sunny, 25\u00b0C"}}', "output.mime_type": "application/json", - "openinference.span.kind": "TOOL" + "openinference.span.kind": "TOOL", }, - "parent_span_id": 5741673860775380214 + "parent_span_id": 5741673860775380214, }, { "name": "call_llm", @@ -278,35 +278,35 @@ "gcp.vertex.agent.invocation_id": "e-b4defdb1-d023-47fa-9962-acd1e95ba1e1", "gcp.vertex.agent.session_id": "veadk_example_session", "gcp.vertex.agent.event_id": "262f18e2-3756-4633-95ee-5e76fe07b754", - "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}]}", - "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_4x60qqjspcwxfu3jtsw1m3sh\",\"args\":{\"city\":\"Beijing\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":316,\"prompt_token_count\":584,\"total_token_count\":900}}", + "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}]}', + "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"function_call":{"id":"call_4x60qqjspcwxfu3jtsw1m3sh","args":{"city":"Beijing"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":316,"prompt_token_count":584,"total_token_count":900}}', "gen_ai.usage.input_tokens": 584, "gen_ai.usage.output_tokens": 900, "llm.provider": "google", - "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}]}", + "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}]}', "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', "llm.input_messages.1.message.role": "user", "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", "llm.input_messages.1.message.contents.0.message_content.type": "text", "llm.input_messages.2.message.role": "model", "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', "llm.input_messages.3.message.role": "tool", "llm.input_messages.3.message.name": "get_city_weather", - "llm.input_messages.3.message.content": "{\"result\": \"Cool, 18\u00b0C\"}", + "llm.input_messages.3.message.content": '{"result": "Cool, 18\u00b0C"}', "llm.input_messages.4.message.role": "model", "llm.input_messages.4.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", "llm.input_messages.4.message.contents.0.message_content.type": "text", "llm.input_messages.5.message.role": "user", "llm.input_messages.5.message.contents.0.message_content.text": "How is the weather like in BeiJing?", "llm.input_messages.5.message.contents.0.message_content.type": "text", - "output.value": "{\"content\":{\"parts\":[{\"function_call\":{\"id\":\"call_4x60qqjspcwxfu3jtsw1m3sh\",\"args\":{\"city\":\"Beijing\"},\"name\":\"get_city_weather\"}}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":316,\"prompt_token_count\":584,\"total_token_count\":900}}", + "output.value": '{"content":{"parts":[{"function_call":{"id":"call_4x60qqjspcwxfu3jtsw1m3sh","args":{"city":"Beijing"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":316,"prompt_token_count":584,"total_token_count":900}}', "output.mime_type": "application/json", "llm.token_count.total": 900, "llm.token_count.prompt": 584, @@ -314,19 +314,19 @@ "llm.output_messages.0.message.role": "model", "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Beijing\"}", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"city": "Beijing"}', "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in BeiJing?\"}]", + "gen_ai.prompt.0.content": '[{"text": "How is the weather like in BeiJing?"}]', "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": "[{\"function_call\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"args\": {\"city\": \"Beijing\"}, \"name\": \"get_city_weather\"}}]", + "gen_ai.completion.0.content": '[{"function_call": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "args": {"city": "Beijing"}, "name": "get_city_weather"}}]', "gen_ai.usage.prompt_tokens": 584, "gen_ai.usage.completion_tokens": 316, "gen_ai.usage.total_tokens": 900, "gen_ai.usage.cache_read_input_tokens": 0, "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM" + "openinference.span.kind": "LLM", }, - "parent_span_id": 8830792093920828159 + "parent_span_id": 8830792093920828159, }, { "name": "call_llm", @@ -342,28 +342,28 @@ "gcp.vertex.agent.invocation_id": "e-b4defdb1-d023-47fa-9962-acd1e95ba1e1", "gcp.vertex.agent.session_id": "veadk_example_session", "gcp.vertex.agent.event_id": "2aad3f22-94da-4e55-9976-56b2256b61fc", - "gcp.vertex.agent.llm_request": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"args\": {\"city\": \"Beijing\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Sunny, 25\u00b0C\"}}}], \"role\": \"user\"}]}", - "gcp.vertex.agent.llm_response": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900}}", + "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "args": {"city": "Beijing"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "name": "get_city_weather", "response": {"result": "Sunny, 25\u00b0C"}}}], "role": "user"}]}', + "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900}}', "gen_ai.usage.input_tokens": 644, "gen_ai.usage.output_tokens": 900, "llm.provider": "google", - "input.value": "{\"model\": \"openai/doubao-seed-1.6-250615\", \"config\": {\"system_instruction\": \"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\", \"tools\": [{\"function_declarations\": [{\"description\": \"Retrieves the weather information of a given city. the args must in English\", \"name\": \"get_city_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"STRING\"}}, \"required\": [\"city\"], \"type\": \"OBJECT\"}}]}], \"labels\": {\"adk_agent_name\": \"chat_robot\"}}, \"contents\": [{\"parts\": [{\"text\": \"How is the weather like in Xi'an?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"args\": {\"city\": \"Xi'an\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_6ow5622pvcouw3tpvr7rfqtl\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Cool, 18\u00b0C\"}}}], \"role\": \"user\"}, {\"parts\": [{\"text\": \"The weather in Xi'an is cool, with a temperature of 18\u00b0C.\"}], \"role\": \"model\"}, {\"parts\": [{\"text\": \"How is the weather like in BeiJing?\"}], \"role\": \"user\"}, {\"parts\": [{\"function_call\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"args\": {\"city\": \"Beijing\"}, \"name\": \"get_city_weather\"}}], \"role\": \"model\"}, {\"parts\": [{\"function_response\": {\"id\": \"call_4x60qqjspcwxfu3jtsw1m3sh\", \"name\": \"get_city_weather\", \"response\": {\"result\": \"Sunny, 25\u00b0C\"}}}], \"role\": \"user\"}]}", + "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "args": {"city": "Beijing"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "name": "get_city_weather", "response": {"result": "Sunny, 25\u00b0C"}}}], "role": "user"}]}', "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": "{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}", + "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": "{\"system_instruction\":\"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\\"chat_robot\\\".\\n\\n The description about you is \\\"A robot talk with user.\\\"\",\"tools\":[{\"function_declarations\":[{\"description\":\"Retrieves the weather information of a given city. the args must in English\",\"name\":\"get_city_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"],\"type\":\"OBJECT\"}}]}],\"labels\":{\"adk_agent_name\":\"chat_robot\"}}", + "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": "Talk with user friendly.\n\nYou are an agent. Your internal name is \"chat_robot\".\n\n The description about you is \"A robot talk with user.\"", + "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', "llm.input_messages.1.message.role": "user", "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", "llm.input_messages.1.message.contents.0.message_content.type": "text", "llm.input_messages.2.message.role": "model", "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Xi'an\"}", + "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', "llm.input_messages.3.message.role": "tool", "llm.input_messages.3.message.name": "get_city_weather", - "llm.input_messages.3.message.content": "{\"result\": \"Cool, 18\u00b0C\"}", + "llm.input_messages.3.message.content": '{"result": "Cool, 18\u00b0C"}', "llm.input_messages.4.message.role": "model", "llm.input_messages.4.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", "llm.input_messages.4.message.contents.0.message_content.type": "text", @@ -373,11 +373,11 @@ "llm.input_messages.6.message.role": "model", "llm.input_messages.6.message.tool_calls.0.tool_call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", "llm.input_messages.6.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.6.message.tool_calls.0.tool_call.function.arguments": "{\"city\": \"Beijing\"}", + "llm.input_messages.6.message.tool_calls.0.tool_call.function.arguments": '{"city": "Beijing"}', "llm.input_messages.7.message.role": "tool", "llm.input_messages.7.message.name": "get_city_weather", - "llm.input_messages.7.message.content": "{\"result\": \"Sunny, 25\u00b0C\"}", - "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900}}", + "llm.input_messages.7.message.content": '{"result": "Sunny, 25\u00b0C"}', + "output.value": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900}}', "output.mime_type": "application/json", "llm.token_count.total": 900, "llm.token_count.prompt": 644, @@ -386,17 +386,17 @@ "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Beijing is sunny, with a temperature of 25\u00b0C.", "llm.output_messages.0.message.contents.0.message_content.type": "text", "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": "[{\"text\": \"How is the weather like in BeiJing?\"}]", + "gen_ai.prompt.0.content": '[{"text": "How is the weather like in BeiJing?"}]', "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": "[{\"text\": \"The weather in Beijing is sunny, with a temperature of 25\\u00b0C.\"}]", + "gen_ai.completion.0.content": '[{"text": "The weather in Beijing is sunny, with a temperature of 25\\u00b0C."}]', "gen_ai.usage.prompt_tokens": 644, "gen_ai.usage.completion_tokens": 256, "gen_ai.usage.total_tokens": 900, "gen_ai.usage.cache_read_input_tokens": 0, "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM" + "openinference.span.kind": "LLM", }, - "parent_span_id": 8830792093920828159 + "parent_span_id": 8830792093920828159, }, { "name": "agent_run [chat_robot]", @@ -411,11 +411,11 @@ "gen_ai.request.model": "openai/doubao-seed-1.6-250615", "gen_ai.response.model": "openai/doubao-seed-1.6-250615", "gen_ai.request.type": "completion", - "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900},\"invocation_id\":\"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"2aad3f22-94da-4e55-9976-56b2256b61fc\",\"timestamp\":1754884684.74225}", + "output.value": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900},"invocation_id":"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"2aad3f22-94da-4e55-9976-56b2256b61fc","timestamp":1754884684.74225}', "output.mime_type": "application/json", - "openinference.span.kind": "AGENT" + "openinference.span.kind": "AGENT", }, - "parent_span_id": 6624741805060007734 + "parent_span_id": 6624741805060007734, }, { "name": "invocation [veadk_default_app]", @@ -424,16 +424,16 @@ "start_time": 1754884677050051000, "end_time": 1754884692281935000, "attributes": { - "input.value": "{\"user_id\": \"veadk_default_user\", \"session_id\": \"veadk_example_session\", \"new_message\": \"{\\\"parts\\\": [{\\\"video_metadata\\\": null, \\\"thought\\\": null, \\\"inline_data\\\": null, \\\"file_data\\\": null, \\\"thought_signature\\\": null, \\\"code_execution_result\\\": null, \\\"executable_code\\\": null, \\\"function_call\\\": null, \\\"function_response\\\": null, \\\"text\\\": \\\"How is the weather like in BeiJing?\\\"}], \\\"role\\\": \\\"user\\\"}\", \"run_config\": \"{\\\"speech_config\\\": null, \\\"response_modalities\\\": null, \\\"save_input_blobs_as_artifacts\\\": false, \\\"support_cfc\\\": false, \\\"streaming_mode\\\": \\\"StreamingMode.NONE\\\", \\\"output_audio_transcription\\\": null, \\\"input_audio_transcription\\\": null, \\\"realtime_input_config\\\": null, \\\"enable_affective_dialog\\\": null, \\\"proactivity\\\": null, \\\"max_llm_calls\\\": 500}\"}", + "input.value": '{"user_id": "veadk_default_user", "session_id": "veadk_example_session", "new_message": "{\\"parts\\": [{\\"video_metadata\\": null, \\"thought\\": null, \\"inline_data\\": null, \\"file_data\\": null, \\"thought_signature\\": null, \\"code_execution_result\\": null, \\"executable_code\\": null, \\"function_call\\": null, \\"function_response\\": null, \\"text\\": \\"How is the weather like in BeiJing?\\"}], \\"role\\": \\"user\\"}", "run_config": "{\\"speech_config\\": null, \\"response_modalities\\": null, \\"save_input_blobs_as_artifacts\\": false, \\"support_cfc\\": false, \\"streaming_mode\\": \\"StreamingMode.NONE\\", \\"output_audio_transcription\\": null, \\"input_audio_transcription\\": null, \\"realtime_input_config\\": null, \\"enable_affective_dialog\\": null, \\"proactivity\\": null, \\"max_llm_calls\\": 500}"}', "input.mime_type": "application/json", "user.id": "veadk_default_user", "session.id": "veadk_example_session", - "output.value": "{\"content\":{\"parts\":[{\"text\":\"The weather in Beijing is sunny, with a temperature of 25\u00b0C.\"}],\"role\":\"model\"},\"partial\":false,\"usage_metadata\":{\"candidates_token_count\":256,\"prompt_token_count\":644,\"total_token_count\":900},\"invocation_id\":\"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1\",\"author\":\"chat_robot\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{}},\"id\":\"2aad3f22-94da-4e55-9976-56b2256b61fc\",\"timestamp\":1754884684.74225}", + "output.value": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900},"invocation_id":"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"2aad3f22-94da-4e55-9976-56b2256b61fc","timestamp":1754884684.74225}', "output.mime_type": "application/json", - "openinference.span.kind": "CHAIN" + "openinference.span.kind": "CHAIN", }, - "parent_span_id": None - } + "parent_span_id": None, + }, ] @@ -465,7 +465,9 @@ def test_evaluator_using_tracing(): with open(tracing_set_file_path, "w") as f: json.dump(TRACE_SET_DATA, f) - base_evaluator.generate_eval_data_from_tracing(tracing_set_file_path=tracing_set_file_path) + base_evaluator.generate_eval_data_from_tracing( + tracing_set_file_path=tracing_set_file_path + ) assert len(base_evaluator.invocation_list) == 1 assert len(base_evaluator.invocation_list[0].invocations) == 2 @@ -474,4 +476,4 @@ def test_evaluator_using_tracing(): == "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441" ) - os.remove(tracing_set_file_path) \ No newline at end of file + os.remove(tracing_set_file_path) diff --git a/veadk/evaluation/base_evaluator.py b/veadk/evaluation/base_evaluator.py index 18f359ac..257f0e1b 100644 --- a/veadk/evaluation/base_evaluator.py +++ b/veadk/evaluation/base_evaluator.py @@ -114,48 +114,69 @@ def load_tracing_set(self, tracing_set_file_path: str) -> EvalSet: # Extract tool_uses from spans with name starting with "execute_tool" for span in spans: if span["name"].startswith("execute_tool"): - tool_uses.append({ - "id": span["attributes"].get("gen_ai.tool.call.id", None), - "args": json.loads(span["attributes"].get("gcp.vertex.agent.tool_call_args", "{}")), - "name": span["attributes"].get("gen_ai.tool.name", None), - }) + tool_uses.append( + { + "id": span["attributes"].get("gen_ai.tool.call.id", None), + "args": json.loads( + span["attributes"].get( + "gcp.vertex.agent.tool_call_args", "{}" + ) + ), + "name": span["attributes"].get("gen_ai.tool.name", None), + } + ) # Extract conversation data from spans with name starting with "invocation" for span in spans: if span["name"].startswith("invocation"): # Parse input.value and output.value as JSON - input_value = json.loads(span["attributes"].get("input.value", "{}")) - output_value = json.loads(span["attributes"].get("output.value", "{}")) + input_value = json.loads( + span["attributes"].get("input.value", "{}") + ) + output_value = json.loads( + span["attributes"].get("output.value", "{}") + ) user_content = json.loads(input_value.get("new_message", {})) final_response = json.loads(json.dumps(user_content)) - final_response["parts"][0]["text"] = output_value.get("content", {}).get("parts", [{}])[0].get("text", None) + final_response["parts"][0]["text"] = ( + output_value.get("content", {}) + .get("parts", [{}])[0] + .get("text", None) + ) final_response["role"] = None - conversation.append({ - "invocation_id": output_value.get("invocation_id", str(uuid.uuid4())), - "user_content": user_content, - "final_response": final_response, - "intermediate_data": { - "tool_uses": tool_uses, - "intermediate_responses": [] - }, - "creation_timestamp": span["start_time"] / 1e9, - }) + conversation.append( + { + "invocation_id": output_value.get( + "invocation_id", str(uuid.uuid4()) + ), + "user_content": user_content, + "final_response": final_response, + "intermediate_data": { + "tool_uses": tool_uses, + "intermediate_responses": [], + }, + "creation_timestamp": span["start_time"] / 1e9, + } + ) user_id = input_value.get("user_id", None) - app_name = span["name"].replace("invocation", "").strip().strip("[]") + app_name = ( + span["name"].replace("invocation", "").strip().strip("[]") + ) creation_timestamp = span["start_time"] / 1e9 - - eval_cases.append({ - "eval_id": f"veadk_eval_{formatted_timestamp()}", - "conversation": conversation, - "session_input": { - "app_name": app_name, - "user_id": user_id, - "state": {}, - }, - "creation_timestamp": creation_timestamp, - }) + eval_cases.append( + { + "eval_id": f"veadk_eval_{formatted_timestamp()}", + "conversation": conversation, + "session_input": { + "app_name": app_name, + "user_id": user_id, + "state": {}, + }, + "creation_timestamp": creation_timestamp, + } + ) evalset = EvalSet( eval_set_id="default", @@ -164,15 +185,17 @@ def load_tracing_set(self, tracing_set_file_path: str) -> EvalSet: eval_cases=eval_cases, creation_timestamp=creation_timestamp, ) - + return evalset def generate_eval_data_from_tracing(self, tracing_set_file_path: str): eval_case_data_list: list[EvalCaseData] = [] eval_cases = self.load_tracing_set(tracing_set_file_path).eval_cases self.generate_invocation_data(eval_cases, eval_case_data_list) - - def generate_invocation_data(self, eval_cases: list[EvalSet], eval_case_data_list: list[EvalCaseData]): + + def generate_invocation_data( + self, eval_cases: list[EvalSet], eval_case_data_list: list[EvalCaseData] + ): for eval_case in eval_cases: eval_case_data = EvalCaseData(invocations=[]) self.agent_information_list.append( From 15e3af5fbfec9eb837c73f764593cc9b571fa7a2 Mon Sep 17 00:00:00 2001 From: "wuqingfu.528" Date: Mon, 11 Aug 2025 20:51:59 +0800 Subject: [PATCH 3/5] fix: shorten tracing samples in the test --- tests/test_evaluator.py | 318 +--------------------------------------- 1 file changed, 1 insertion(+), 317 deletions(-) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index de1878f0..6394abc4 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -67,26 +67,10 @@ "start_time": 1754884672226444000, "end_time": 1754884672226993000, "attributes": { - "session.id": "veadk_example_session", - "user.id": "veadk_default_user", - "gen_ai.system": "gcp.vertex.agent", - "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "get_city_weather", "gen_ai.tool.description": "Retrieves the weather information of a given city. the args must in English", "gen_ai.tool.call.id": "call_6ow5622pvcouw3tpvr7rfqtl", "gcp.vertex.agent.tool_call_args": '{"city": "Xi\'an"}', - "gcp.vertex.agent.event_id": "afdc4159-10f1-4f94-aa91-1003a9b6f1cc", - "gcp.vertex.agent.tool_response": '{"result": "Cool, 18\u00b0C"}', - "gcp.vertex.agent.llm_request": "{}", - "gcp.vertex.agent.llm_response": "{}", - "tool.name": "get_city_weather", - "tool.description": "Retrieves the weather information of a given city. the args must in English", - "tool.parameters": '{"city": "Xi\'an"}', - "input.value": '{"city": "Xi\'an"}', - "input.mime_type": "application/json", - "output.value": '{"id":"call_6ow5622pvcouw3tpvr7rfqtl","name":"get_city_weather","response":{"result":"Cool, 18\u00b0C"}}', - "output.mime_type": "application/json", - "openinference.span.kind": "TOOL", }, "parent_span_id": 7997784243558253239, }, @@ -94,50 +78,9 @@ "name": "call_llm", "span_id": 7997784243558253239, "trace_id": 115143748123782151752771111946932434777, - "start_time": 1754884660688448000, - "end_time": 1754884672227098000, "attributes": { "session.id": "veadk_example_session", "user.id": "veadk_default_user", - "gen_ai.system": "gcp.vertex.agent", - "gen_ai.request.model": "openai/doubao-seed-1.6-250615", - "gcp.vertex.agent.invocation_id": "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441", - "gcp.vertex.agent.session_id": "veadk_example_session", - "gcp.vertex.agent.event_id": "9e94cb99-9f78-47db-a79c-f5422354953e", - "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}]}', - "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"function_call":{"id":"call_6ow5622pvcouw3tpvr7rfqtl","args":{"city":"Xi\'an"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":496,"prompt_token_count":487,"total_token_count":983}}', - "gen_ai.usage.input_tokens": 487, - "gen_ai.usage.output_tokens": 983, - "llm.provider": "google", - "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}]}', - "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', - "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', - "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', - "llm.input_messages.1.message.role": "user", - "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", - "llm.input_messages.1.message.contents.0.message_content.type": "text", - "output.value": '{"content":{"parts":[{"function_call":{"id":"call_6ow5622pvcouw3tpvr7rfqtl","args":{"city":"Xi\'an"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":496,"prompt_token_count":487,"total_token_count":983}}', - "output.mime_type": "application/json", - "llm.token_count.total": 983, - "llm.token_count.prompt": 487, - "llm.token_count.completion": 496, - "llm.output_messages.0.message.role": "model", - "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", - "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', - "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": '[{"text": "How is the weather like in Xi\'an?"}]', - "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": '[{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}]', - "gen_ai.usage.prompt_tokens": 487, - "gen_ai.usage.completion_tokens": 496, - "gen_ai.usage.total_tokens": 983, - "gen_ai.usage.cache_read_input_tokens": 0, - "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM", }, "parent_span_id": 14844888006539887900, }, @@ -145,56 +88,9 @@ "name": "call_llm", "span_id": 7789424022423491416, "trace_id": 115143748123782151752771111946932434777, - "start_time": 1754884672227703000, - "end_time": 1754884676664423000, "attributes": { "session.id": "veadk_example_session", "user.id": "veadk_default_user", - "gen_ai.system": "gcp.vertex.agent", - "gen_ai.request.model": "openai/doubao-seed-1.6-250615", - "gcp.vertex.agent.invocation_id": "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441", - "gcp.vertex.agent.session_id": "veadk_example_session", - "gcp.vertex.agent.event_id": "c0929124-9be0-4f75-a6ba-f7a531c9ccb6", - "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}]}', - "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679}}', - "gen_ai.usage.input_tokens": 547, - "gen_ai.usage.output_tokens": 679, - "llm.provider": "google", - "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}]}', - "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', - "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', - "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', - "llm.input_messages.1.message.role": "user", - "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", - "llm.input_messages.1.message.contents.0.message_content.type": "text", - "llm.input_messages.2.message.role": "model", - "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', - "llm.input_messages.3.message.role": "tool", - "llm.input_messages.3.message.name": "get_city_weather", - "llm.input_messages.3.message.content": '{"result": "Cool, 18\u00b0C"}', - "output.value": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679}}', - "output.mime_type": "application/json", - "llm.token_count.total": 679, - "llm.token_count.prompt": 547, - "llm.token_count.completion": 132, - "llm.output_messages.0.message.role": "model", - "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", - "llm.output_messages.0.message.contents.0.message_content.type": "text", - "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": '[{"text": "How is the weather like in Xi\'an?"}]', - "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": '[{"text": "The weather in Xi\'an is cool, with a temperature of 18\\u00b0C."}]', - "gen_ai.usage.prompt_tokens": 547, - "gen_ai.usage.completion_tokens": 132, - "gen_ai.usage.total_tokens": 679, - "gen_ai.usage.cache_read_input_tokens": 0, - "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM", }, "parent_span_id": 14844888006539887900, }, @@ -202,18 +98,9 @@ "name": "agent_run [chat_robot]", "span_id": 14844888006539887900, "trace_id": 115143748123782151752771111946932434777, - "start_time": 1754884660688136000, - "end_time": 1754884676664691000, "attributes": { "session.id": "veadk_example_session", "user.id": "veadk_default_user", - "gen_ai.system": "veadk", - "gen_ai.request.model": "openai/doubao-seed-1.6-250615", - "gen_ai.response.model": "openai/doubao-seed-1.6-250615", - "gen_ai.request.type": "completion", - "output.value": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679},"invocation_id":"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"c0929124-9be0-4f75-a6ba-f7a531c9ccb6","timestamp":1754884672.227546}', - "output.mime_type": "application/json", - "openinference.span.kind": "AGENT", }, "parent_span_id": 2943363177785645047, }, @@ -225,212 +112,9 @@ "end_time": 1754884676664833000, "attributes": { "input.value": '{"user_id": "veadk_default_user", "session_id": "veadk_example_session", "new_message": "{\\"parts\\": [{\\"video_metadata\\": null, \\"thought\\": null, \\"inline_data\\": null, \\"file_data\\": null, \\"thought_signature\\": null, \\"code_execution_result\\": null, \\"executable_code\\": null, \\"function_call\\": null, \\"function_response\\": null, \\"text\\": \\"How is the weather like in Xi\'an?\\"}], \\"role\\": \\"user\\"}", "run_config": "{\\"speech_config\\": null, \\"response_modalities\\": null, \\"save_input_blobs_as_artifacts\\": false, \\"support_cfc\\": false, \\"streaming_mode\\": \\"StreamingMode.NONE\\", \\"output_audio_transcription\\": null, \\"input_audio_transcription\\": null, \\"realtime_input_config\\": null, \\"enable_affective_dialog\\": null, \\"proactivity\\": null, \\"max_llm_calls\\": 500}"}', - "input.mime_type": "application/json", "user.id": "veadk_default_user", "session.id": "veadk_example_session", "output.value": '{"content":{"parts":[{"text":"The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":132,"prompt_token_count":547,"total_token_count":679},"invocation_id":"e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"c0929124-9be0-4f75-a6ba-f7a531c9ccb6","timestamp":1754884672.227546}', - "output.mime_type": "application/json", - "openinference.span.kind": "CHAIN", - }, - "parent_span_id": None, - }, - { - "name": "execute_tool get_city_weather", - "span_id": 10315930087681041331, - "trace_id": 225149951093293230286974602372571218832, - "start_time": 1754884684740875000, - "end_time": 1754884684741386000, - "attributes": { - "session.id": "veadk_example_session", - "user.id": "veadk_default_user", - "gen_ai.system": "gcp.vertex.agent", - "gen_ai.operation.name": "execute_tool", - "gen_ai.tool.name": "get_city_weather", - "gen_ai.tool.description": "Retrieves the weather information of a given city. the args must in English", - "gen_ai.tool.call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", - "gcp.vertex.agent.tool_call_args": '{"city": "Beijing"}', - "gcp.vertex.agent.event_id": "e4e1b9cb-c147-4fb7-8cf6-dfecc0e0b9a0", - "gcp.vertex.agent.tool_response": '{"result": "Sunny, 25\u00b0C"}', - "gcp.vertex.agent.llm_request": "{}", - "gcp.vertex.agent.llm_response": "{}", - "tool.name": "get_city_weather", - "tool.description": "Retrieves the weather information of a given city. the args must in English", - "tool.parameters": '{"city": "Beijing"}', - "input.value": '{"city": "Beijing"}', - "input.mime_type": "application/json", - "output.value": '{"id":"call_4x60qqjspcwxfu3jtsw1m3sh","name":"get_city_weather","response":{"result":"Sunny, 25\u00b0C"}}', - "output.mime_type": "application/json", - "openinference.span.kind": "TOOL", - }, - "parent_span_id": 5741673860775380214, - }, - { - "name": "call_llm", - "span_id": 5741673860775380214, - "trace_id": 225149951093293230286974602372571218832, - "start_time": 1754884677052310000, - "end_time": 1754884684741523000, - "attributes": { - "session.id": "veadk_example_session", - "user.id": "veadk_default_user", - "gen_ai.system": "gcp.vertex.agent", - "gen_ai.request.model": "openai/doubao-seed-1.6-250615", - "gcp.vertex.agent.invocation_id": "e-b4defdb1-d023-47fa-9962-acd1e95ba1e1", - "gcp.vertex.agent.session_id": "veadk_example_session", - "gcp.vertex.agent.event_id": "262f18e2-3756-4633-95ee-5e76fe07b754", - "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}]}', - "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"function_call":{"id":"call_4x60qqjspcwxfu3jtsw1m3sh","args":{"city":"Beijing"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":316,"prompt_token_count":584,"total_token_count":900}}', - "gen_ai.usage.input_tokens": 584, - "gen_ai.usage.output_tokens": 900, - "llm.provider": "google", - "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}]}', - "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', - "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', - "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', - "llm.input_messages.1.message.role": "user", - "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", - "llm.input_messages.1.message.contents.0.message_content.type": "text", - "llm.input_messages.2.message.role": "model", - "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', - "llm.input_messages.3.message.role": "tool", - "llm.input_messages.3.message.name": "get_city_weather", - "llm.input_messages.3.message.content": '{"result": "Cool, 18\u00b0C"}', - "llm.input_messages.4.message.role": "model", - "llm.input_messages.4.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", - "llm.input_messages.4.message.contents.0.message_content.type": "text", - "llm.input_messages.5.message.role": "user", - "llm.input_messages.5.message.contents.0.message_content.text": "How is the weather like in BeiJing?", - "llm.input_messages.5.message.contents.0.message_content.type": "text", - "output.value": '{"content":{"parts":[{"function_call":{"id":"call_4x60qqjspcwxfu3jtsw1m3sh","args":{"city":"Beijing"},"name":"get_city_weather"}}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":316,"prompt_token_count":584,"total_token_count":900}}', - "output.mime_type": "application/json", - "llm.token_count.total": 900, - "llm.token_count.prompt": 584, - "llm.token_count.completion": 316, - "llm.output_messages.0.message.role": "model", - "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", - "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"city": "Beijing"}', - "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": '[{"text": "How is the weather like in BeiJing?"}]', - "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": '[{"function_call": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "args": {"city": "Beijing"}, "name": "get_city_weather"}}]', - "gen_ai.usage.prompt_tokens": 584, - "gen_ai.usage.completion_tokens": 316, - "gen_ai.usage.total_tokens": 900, - "gen_ai.usage.cache_read_input_tokens": 0, - "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM", - }, - "parent_span_id": 8830792093920828159, - }, - { - "name": "call_llm", - "span_id": 1294773233164416528, - "trace_id": 225149951093293230286974602372571218832, - "start_time": 1754884684742479000, - "end_time": 1754884692281442000, - "attributes": { - "session.id": "veadk_example_session", - "user.id": "veadk_default_user", - "gen_ai.system": "gcp.vertex.agent", - "gen_ai.request.model": "openai/doubao-seed-1.6-250615", - "gcp.vertex.agent.invocation_id": "e-b4defdb1-d023-47fa-9962-acd1e95ba1e1", - "gcp.vertex.agent.session_id": "veadk_example_session", - "gcp.vertex.agent.event_id": "2aad3f22-94da-4e55-9976-56b2256b61fc", - "gcp.vertex.agent.llm_request": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "args": {"city": "Beijing"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "name": "get_city_weather", "response": {"result": "Sunny, 25\u00b0C"}}}], "role": "user"}]}', - "gcp.vertex.agent.llm_response": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900}}', - "gen_ai.usage.input_tokens": 644, - "gen_ai.usage.output_tokens": 900, - "llm.provider": "google", - "input.value": '{"model": "openai/doubao-seed-1.6-250615", "config": {"system_instruction": "Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"", "tools": [{"function_declarations": [{"description": "Retrieves the weather information of a given city. the args must in English", "name": "get_city_weather", "parameters": {"properties": {"city": {"type": "STRING"}}, "required": ["city"], "type": "OBJECT"}}]}], "labels": {"adk_agent_name": "chat_robot"}}, "contents": [{"parts": [{"text": "How is the weather like in Xi\'an?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "args": {"city": "Xi\'an"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_6ow5622pvcouw3tpvr7rfqtl", "name": "get_city_weather", "response": {"result": "Cool, 18\u00b0C"}}}], "role": "user"}, {"parts": [{"text": "The weather in Xi\'an is cool, with a temperature of 18\u00b0C."}], "role": "model"}, {"parts": [{"text": "How is the weather like in BeiJing?"}], "role": "user"}, {"parts": [{"function_call": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "args": {"city": "Beijing"}, "name": "get_city_weather"}}], "role": "model"}, {"parts": [{"function_response": {"id": "call_4x60qqjspcwxfu3jtsw1m3sh", "name": "get_city_weather", "response": {"result": "Sunny, 25\u00b0C"}}}], "role": "user"}]}', - "input.mime_type": "application/json", - "llm.tools.0.tool.json_schema": '{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}', - "llm.model_name": "openai/doubao-seed-1.6-250615", - "llm.invocation_parameters": '{"system_instruction":"Talk with user friendly.\\n\\nYou are an agent. Your internal name is \\"chat_robot\\".\\n\\n The description about you is \\"A robot talk with user.\\"","tools":[{"function_declarations":[{"description":"Retrieves the weather information of a given city. the args must in English","name":"get_city_weather","parameters":{"properties":{"city":{"type":"STRING"}},"required":["city"],"type":"OBJECT"}}]}],"labels":{"adk_agent_name":"chat_robot"}}', - "llm.input_messages.0.message.role": "system", - "llm.input_messages.0.message.content": 'Talk with user friendly.\n\nYou are an agent. Your internal name is "chat_robot".\n\n The description about you is "A robot talk with user."', - "llm.input_messages.1.message.role": "user", - "llm.input_messages.1.message.contents.0.message_content.text": "How is the weather like in Xi'an?", - "llm.input_messages.1.message.contents.0.message_content.type": "text", - "llm.input_messages.2.message.role": "model", - "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_6ow5622pvcouw3tpvr7rfqtl", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": '{"city": "Xi\'an"}', - "llm.input_messages.3.message.role": "tool", - "llm.input_messages.3.message.name": "get_city_weather", - "llm.input_messages.3.message.content": '{"result": "Cool, 18\u00b0C"}', - "llm.input_messages.4.message.role": "model", - "llm.input_messages.4.message.contents.0.message_content.text": "The weather in Xi'an is cool, with a temperature of 18\u00b0C.", - "llm.input_messages.4.message.contents.0.message_content.type": "text", - "llm.input_messages.5.message.role": "user", - "llm.input_messages.5.message.contents.0.message_content.text": "How is the weather like in BeiJing?", - "llm.input_messages.5.message.contents.0.message_content.type": "text", - "llm.input_messages.6.message.role": "model", - "llm.input_messages.6.message.tool_calls.0.tool_call.id": "call_4x60qqjspcwxfu3jtsw1m3sh", - "llm.input_messages.6.message.tool_calls.0.tool_call.function.name": "get_city_weather", - "llm.input_messages.6.message.tool_calls.0.tool_call.function.arguments": '{"city": "Beijing"}', - "llm.input_messages.7.message.role": "tool", - "llm.input_messages.7.message.name": "get_city_weather", - "llm.input_messages.7.message.content": '{"result": "Sunny, 25\u00b0C"}', - "output.value": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900}}', - "output.mime_type": "application/json", - "llm.token_count.total": 900, - "llm.token_count.prompt": 644, - "llm.token_count.completion": 256, - "llm.output_messages.0.message.role": "model", - "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Beijing is sunny, with a temperature of 25\u00b0C.", - "llm.output_messages.0.message.contents.0.message_content.type": "text", - "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": '[{"text": "How is the weather like in BeiJing?"}]', - "gen_ai.completion.0.role": "model", - "gen_ai.completion.0.content": '[{"text": "The weather in Beijing is sunny, with a temperature of 25\\u00b0C."}]', - "gen_ai.usage.prompt_tokens": 644, - "gen_ai.usage.completion_tokens": 256, - "gen_ai.usage.total_tokens": 900, - "gen_ai.usage.cache_read_input_tokens": 0, - "gen_ai.usage.cache_create_input_tokens": 0, - "openinference.span.kind": "LLM", - }, - "parent_span_id": 8830792093920828159, - }, - { - "name": "agent_run [chat_robot]", - "span_id": 8830792093920828159, - "trace_id": 225149951093293230286974602372571218832, - "start_time": 1754884677050889000, - "end_time": 1754884692281791000, - "attributes": { - "session.id": "veadk_example_session", - "user.id": "veadk_default_user", - "gen_ai.system": "veadk", - "gen_ai.request.model": "openai/doubao-seed-1.6-250615", - "gen_ai.response.model": "openai/doubao-seed-1.6-250615", - "gen_ai.request.type": "completion", - "output.value": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900},"invocation_id":"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"2aad3f22-94da-4e55-9976-56b2256b61fc","timestamp":1754884684.74225}', - "output.mime_type": "application/json", - "openinference.span.kind": "AGENT", - }, - "parent_span_id": 6624741805060007734, - }, - { - "name": "invocation [veadk_default_app]", - "span_id": 6624741805060007734, - "trace_id": 225149951093293230286974602372571218832, - "start_time": 1754884677050051000, - "end_time": 1754884692281935000, - "attributes": { - "input.value": '{"user_id": "veadk_default_user", "session_id": "veadk_example_session", "new_message": "{\\"parts\\": [{\\"video_metadata\\": null, \\"thought\\": null, \\"inline_data\\": null, \\"file_data\\": null, \\"thought_signature\\": null, \\"code_execution_result\\": null, \\"executable_code\\": null, \\"function_call\\": null, \\"function_response\\": null, \\"text\\": \\"How is the weather like in BeiJing?\\"}], \\"role\\": \\"user\\"}", "run_config": "{\\"speech_config\\": null, \\"response_modalities\\": null, \\"save_input_blobs_as_artifacts\\": false, \\"support_cfc\\": false, \\"streaming_mode\\": \\"StreamingMode.NONE\\", \\"output_audio_transcription\\": null, \\"input_audio_transcription\\": null, \\"realtime_input_config\\": null, \\"enable_affective_dialog\\": null, \\"proactivity\\": null, \\"max_llm_calls\\": 500}"}', - "input.mime_type": "application/json", - "user.id": "veadk_default_user", - "session.id": "veadk_example_session", - "output.value": '{"content":{"parts":[{"text":"The weather in Beijing is sunny, with a temperature of 25\u00b0C."}],"role":"model"},"partial":false,"usage_metadata":{"candidates_token_count":256,"prompt_token_count":644,"total_token_count":900},"invocation_id":"e-b4defdb1-d023-47fa-9962-acd1e95ba1e1","author":"chat_robot","actions":{"state_delta":{},"artifact_delta":{},"requested_auth_configs":{}},"id":"2aad3f22-94da-4e55-9976-56b2256b61fc","timestamp":1754884684.74225}', - "output.mime_type": "application/json", - "openinference.span.kind": "CHAIN", }, "parent_span_id": None, }, @@ -470,7 +154,7 @@ def test_evaluator_using_tracing(): ) assert len(base_evaluator.invocation_list) == 1 - assert len(base_evaluator.invocation_list[0].invocations) == 2 + assert len(base_evaluator.invocation_list[0].invocations) == 1 assert ( base_evaluator.invocation_list[0].invocations[0].invocation_id == "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441" From e82f8c00c91b38458aeb9b64e365e7fa7643e4a1 Mon Sep 17 00:00:00 2001 From: "wuqingfu.528" Date: Tue, 12 Aug 2025 10:02:33 +0800 Subject: [PATCH 4/5] fix: manage evalset and tracing file uniformly in one function --- tests/test_evaluator.py | 14 ++++----- veadk/evaluation/base_evaluator.py | 47 +++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 6394abc4..f48529ec 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -129,7 +129,7 @@ def test_evaluator(): with open(eval_set_file_path, "w") as f: json.dump(EVAL_SET_DATA, f) - base_evaluator.generate_eval_data(eval_set_file_path=eval_set_file_path) + base_evaluator.generate_eval_data(file_path=eval_set_file_path) assert len(base_evaluator.invocation_list) == 1 assert len(base_evaluator.invocation_list[0].invocations) == 1 @@ -141,17 +141,15 @@ def test_evaluator(): os.remove(eval_set_file_path) -def test_evaluator_using_tracing(): +def test_tracing_file_to_evalset(): base_evaluator = BaseEvaluator(agent=None, name="test_evaluator") # save data to file - tracing_set_file_path = "./tracing_set_for_test_evaluator.json" - with open(tracing_set_file_path, "w") as f: + tracing_file_path = "./tracing_for_test_evaluator.json" + with open(tracing_file_path, "w") as f: json.dump(TRACE_SET_DATA, f) - base_evaluator.generate_eval_data_from_tracing( - tracing_set_file_path=tracing_set_file_path - ) + base_evaluator.generate_eval_data(file_path=tracing_file_path) assert len(base_evaluator.invocation_list) == 1 assert len(base_evaluator.invocation_list[0].invocations) == 1 @@ -160,4 +158,4 @@ def test_evaluator_using_tracing(): == "e-ea6bb35b-c3f0-4c5c-b127-c71c7d6d6441" ) - os.remove(tracing_set_file_path) + os.remove(tracing_file_path) diff --git a/veadk/evaluation/base_evaluator.py b/veadk/evaluation/base_evaluator.py index 257f0e1b..06591cd7 100644 --- a/veadk/evaluation/base_evaluator.py +++ b/veadk/evaluation/base_evaluator.py @@ -87,14 +87,14 @@ def load_eval_set(self, eval_set_file: str) -> EvalSet: return load_eval_set_from_file(eval_set_file) - def generate_eval_data(self, eval_set_file_path: str): - eval_case_data_list: list[EvalCaseData] = [] - eval_cases = self.load_eval_set(eval_set_file_path).eval_cases - self.generate_invocation_data(eval_cases, eval_case_data_list) - - def load_tracing_set(self, tracing_set_file_path: str) -> EvalSet: - with open(tracing_set_file_path, "r") as f: - tracing_data = json.load(f) + def load_eval_set_from_tracing(self, tracing_file: str) -> EvalSet: + try: + with open(tracing_file, "r") as f: + tracing_data = json.load(f) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON format in file {tracing_file}: {e}") + except Exception as e: + raise ValueError(f"Error reading file {tracing_file}: {e}") # Group spans by trace_id trace_groups = {} @@ -188,14 +188,33 @@ def load_tracing_set(self, tracing_set_file_path: str) -> EvalSet: return evalset - def generate_eval_data_from_tracing(self, tracing_set_file_path: str): + def generate_eval_data(self, file_path: str): + """Generate evaluation data from a given file and assign it to the class attribute `invocation_list`.""" eval_case_data_list: list[EvalCaseData] = [] - eval_cases = self.load_tracing_set(tracing_set_file_path).eval_cases - self.generate_invocation_data(eval_cases, eval_case_data_list) - def generate_invocation_data( - self, eval_cases: list[EvalSet], eval_case_data_list: list[EvalCaseData] - ): + try: + with open(file_path, "r") as f: + file_content = json.load(f) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON format in file {file_path}: {e}") + except Exception as e: + raise ValueError(f"Error reading file {file_path}: {e}") + + if isinstance(file_content, dict) and "eval_cases" in file_content: + eval_cases = self.load_eval_set(file_path).eval_cases + elif ( + isinstance(file_content, list) + and len(file_content) > 0 + and all( + isinstance(span, dict) and "trace_id" in span for span in file_content + ) + ): + eval_cases = self.load_eval_set_from_tracing(file_path).eval_cases + else: + raise ValueError( + f"Unsupported file format in {file_path}. Please provide a valid file." + ) + for eval_case in eval_cases: eval_case_data = EvalCaseData(invocations=[]) self.agent_information_list.append( From 268b903906f9af966d44a75fb79a7147c86f9d63 Mon Sep 17 00:00:00 2001 From: "wuqingfu.528" Date: Tue, 12 Aug 2025 10:35:40 +0800 Subject: [PATCH 5/5] fix: modify the function name --- veadk/evaluation/base_evaluator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/veadk/evaluation/base_evaluator.py b/veadk/evaluation/base_evaluator.py index 06591cd7..84f87045 100644 --- a/veadk/evaluation/base_evaluator.py +++ b/veadk/evaluation/base_evaluator.py @@ -82,12 +82,12 @@ def __init__( self.result_list: list[EvalResultData] = [] self.agent_information_list: list[dict] = [] - def load_eval_set(self, eval_set_file: str) -> EvalSet: + def _load_eval_set(self, eval_set_file: str) -> EvalSet: from .eval_set_file_loader import load_eval_set_from_file return load_eval_set_from_file(eval_set_file) - def load_eval_set_from_tracing(self, tracing_file: str) -> EvalSet: + def _load_eval_set_from_tracing(self, tracing_file: str) -> EvalSet: try: with open(tracing_file, "r") as f: tracing_data = json.load(f) @@ -201,7 +201,7 @@ def generate_eval_data(self, file_path: str): raise ValueError(f"Error reading file {file_path}: {e}") if isinstance(file_content, dict) and "eval_cases" in file_content: - eval_cases = self.load_eval_set(file_path).eval_cases + eval_cases = self._load_eval_set(file_path).eval_cases elif ( isinstance(file_content, list) and len(file_content) > 0 @@ -209,7 +209,7 @@ def generate_eval_data(self, file_path: str): isinstance(span, dict) and "trace_id" in span for span in file_content ) ): - eval_cases = self.load_eval_set_from_tracing(file_path).eval_cases + eval_cases = self._load_eval_set_from_tracing(file_path).eval_cases else: raise ValueError( f"Unsupported file format in {file_path}. Please provide a valid file."