From 1a7b5120a77994b5f428becd087565a96764f06e Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 16 Jun 2026 11:50:57 -0700 Subject: [PATCH 1/5] Google ADK Tool Instrumentation --- newrelic/config.py | 5 + newrelic/hooks/mlmodel_googleadk.py | 167 +++++++++++++ tests/mlmodel_googleadk/_test_tools.py | 75 ++++++ tests/mlmodel_googleadk/cassette.yaml | 259 +++++++++++++++----- tests/mlmodel_googleadk/conftest.py | 13 +- tests/mlmodel_googleadk/test_tools.py | 113 +++++++++ tests/mlmodel_googleadk/test_tools_error.py | 56 +++++ 7 files changed, 622 insertions(+), 66 deletions(-) create mode 100644 newrelic/hooks/mlmodel_googleadk.py create mode 100644 tests/mlmodel_googleadk/_test_tools.py create mode 100644 tests/mlmodel_googleadk/test_tools.py create mode 100644 tests/mlmodel_googleadk/test_tools_error.py diff --git a/newrelic/config.py b/newrelic/config.py index 6dde4c6bb..04e464e6f 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3185,6 +3185,11 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_autogen", "instrument_autogen_agentchat_agents__assistant_agent", ) + _process_module_definition( + "google.adk.flows.llm_flows.functions", + "newrelic.hooks.mlmodel_googleadk", + "instrument_googleadk_flows_llm_flows_functions", + ) _process_module_definition( "strands.agent.agent", "newrelic.hooks.mlmodel_strands", "instrument_strands_agent_agent" ) diff --git a/newrelic/hooks/mlmodel_googleadk.py b/newrelic/hooks/mlmodel_googleadk.py new file mode 100644 index 000000000..54db81112 --- /dev/null +++ b/newrelic/hooks/mlmodel_googleadk.py @@ -0,0 +1,167 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import sys +import uuid + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.llm_utils import _get_llm_metadata +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings + +_logger = logging.getLogger(__name__) +GOOGLEADK_VERSION = get_package_version("google-adk") + +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Google ADK instrumentation: Failed to record LLM events. Please report this issue to New Relic Support." +TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in Google ADK instrumentation: Failed to extract tool information. If the issue persists, report this issue to New Relic support.\n" + + +async def wrap__execute_single_function_call_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + transaction.add_ml_model_info("GoogleADK", GOOGLEADK_VERSION) + transaction._add_agent_attribute("llm", True) + + tool_name = "tool" + run_id = "" + tool_input = None + agent_name = "agent" + try: + bound_args = bind_args(wrapped, args, kwargs) + function_call = bound_args.get("function_call") + agent = bound_args.get("agent") + if function_call is not None: + tool_name = getattr(function_call, "name", "tool") or "tool" + run_id = getattr(function_call, "id", "") or "" + tool_input = getattr(function_call, "args", None) + if agent is not None: + agent_name = getattr(agent, "name", "agent") or "agent" + except Exception: + _logger.warning(TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) + + func_name = callable_name(wrapped) + function_trace_name = f"{func_name}/{tool_name}" + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/tool/GoogleADK") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + linking_metadata = get_trace_linking_metadata() + tool_id = str(uuid.uuid4()) + + try: + tool_output = await wrapped(*args, **kwargs) + except Exception: + ft.notice_error(attributes={"tool_id": tool_id}) + ft.__exit__(*sys.exc_info()) + try: + tool_event_dict = _construct_base_tool_event_dict( + tool_name=tool_name, + tool_id=tool_id, + run_id=run_id, + tool_input=tool_input, + tool_output=None, + agent_name=agent_name, + error=True, + transaction=transaction, + linking_metadata=linking_metadata, + ) + tool_event_dict["duration"] = ft.duration * 1000 + transaction.record_custom_event("LlmTool", tool_event_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + raise + + ft.__exit__(None, None, None) + try: + response_dict = _extract_tool_response_dict(tool_output) + tool_event_dict = _construct_base_tool_event_dict( + tool_name=tool_name, + tool_id=tool_id, + run_id=run_id, + tool_input=tool_input, + tool_output=response_dict, + agent_name=agent_name, + error=False, + transaction=transaction, + linking_metadata=linking_metadata, + ) + tool_event_dict["duration"] = ft.duration * 1000 + transaction.record_custom_event("LlmTool", tool_event_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_output + + +def _extract_tool_response_dict(tool_output): + """Return the dict at content.parts[*].function_response.response, or None.""" + try: + parts = tool_output.content.parts + for part in parts: + function_response = getattr(part, "function_response", None) + if function_response is not None: + return getattr(function_response, "response", None) + except (AttributeError, TypeError): + pass + return None + + +def _construct_base_tool_event_dict( + tool_name, tool_id, run_id, tool_input, tool_output, agent_name, error, transaction, linking_metadata +): + try: + settings = transaction.settings or global_settings() + + tool_event_dict = { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "google_adk", + "ingest_source": "Python", + } + if error: + tool_event_dict["error"] = True + + if settings.ai_monitoring.record_content.enabled: + tool_event_dict["input"] = str(tool_input) if tool_input else None + tool_event_dict["output"] = str(tool_output) if tool_output is not None else None + + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def instrument_googleadk_flows_llm_flows_functions(module): + if hasattr(module, "_execute_single_function_call_async"): + wrap_function_wrapper(module, "_execute_single_function_call_async", wrap__execute_single_function_call_async) diff --git a/tests/mlmodel_googleadk/_test_tools.py b/tests/mlmodel_googleadk/_test_tools.py new file mode 100644 index 000000000..f736d4ac4 --- /dev/null +++ b/tests/mlmodel_googleadk/_test_tools.py @@ -0,0 +1,75 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools + +from _test_agent import AGENT_NAME +from google.adk.tools import FunctionTool + +TOOL_NAME = "get_capital" + + +def get_capital(country: str) -> dict: + """Return the capital of a country.""" + capitals = {"France": "Paris", "Japan": "Tokyo"} + return {"output": capitals.get(country, "Unknown")} + + +@functools.wraps(get_capital) # Make function names match +def _raising_capital(country: str) -> dict: + """Return the capital of a country.""" + raise ValueError("intentional tool failure") + + +get_capital_tool = FunctionTool(func=get_capital) +raising_tool = FunctionTool(func=_raising_capital) + +EXPECTED_TOOL_INPUT_STR = "{'country': 'France'}" +EXPECTED_TOOL_OUTPUT_STR = "{'output': 'Paris'}" + + +def tool_recorded_event(record_content: bool): + base = { + "id": None, + "run_id": None, + "name": TOOL_NAME, + "span_id": None, + "trace_id": "trace-id", + "agent_name": AGENT_NAME, + "vendor": "google_adk", + "ingest_source": "Python", + "duration": None, + } + if record_content: + base["input"] = EXPECTED_TOOL_INPUT_STR + base["output"] = EXPECTED_TOOL_OUTPUT_STR + return [({"type": "LlmTool"}, base)] + + +def tool_recorded_event_error(record_content: bool): + base = { + "id": None, + "run_id": None, + "name": TOOL_NAME, + "span_id": None, + "trace_id": "trace-id", + "agent_name": AGENT_NAME, + "vendor": "google_adk", + "ingest_source": "Python", + "duration": None, + "error": True, + } + if record_content: + base["input"] = EXPECTED_TOOL_INPUT_STR + return [({"type": "LlmTool"}, base)] diff --git a/tests/mlmodel_googleadk/cassette.yaml b/tests/mlmodel_googleadk/cassette.yaml index b76416ff5..0d431d62c 100644 --- a/tests/mlmodel_googleadk/cassette.yaml +++ b/tests/mlmodel_googleadk/cassette.yaml @@ -13,67 +13,200 @@ # limitations under the License. interactions: - - request: - body: - '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role": - "user"}], "systemInstruction": {"parts": [{"text": "Answer the user''s question - in one word.\n\nYou are an agent. Your internal name is \"my_agent\"."}], "role": - "user"}, "generationConfig": {}}' - headers: - Accept: - - "*/*" - Connection: - - keep-alive - Content-Type: - - application/json - Host: - - generativelanguage.googleapis.com - accept-encoding: - - identity - x-goog-api-key: - - XXXXXX - method: POST - uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent - response: - body: - string: - "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": - [\n {\n \"text\": \"Paris\",\n \"thoughtSignature\": - \"EtcDCtQDAQw51sf7SnGSYaq6KQBVSfkaRtVpY09Vb2mvRQE32lJRfHW/wjeScdNCatEqI9919UNu5PtM9Rq3h9x3LhK7Sn6iNaaudS7x0XfSRPeMz6rm85wO7NWUYcNR8Srj6tk3gF5nh4TZhlpk6JRxEBXMJJLmF50Z7akDL3o9j+S0DKaG58tfKAxMRC5V5xRdR/o0XmAL3v6Ba+9ZtIc4Y/4iMmJsQyM3iJK2Bf4YAOPhmjS5tmFMupQl3ZruDKP3FihTqoLioiTUWSlW29+Ya/RK4pu5wuXz1Qy96N+Av+MzvzO98jD68IbeCKmRkKjwFvN68mPSFQAb2wIc6pmgwetkhkuDuXL4wzVDTny8pZEJLwNvrUE5HodOZAntifwvbijfrdkI4OD99n8F96fC1PO0bnRk1KgRMFEyAmAF+LiZZRSL4bJ2vWbWZM4gTC2VS4ajThqHHu8vwsR6HZh0pnDvzkVF3E7+wbeA+9NKindJ0AsEIX8Wh9S/IdZePv0ojDnUKwGVwC1MA5Z3ZE9KVsfluA3PdrTVSE4pXSP8Hmo4yIF8w3VCiP99iIsDdcgoXr2PgDlDc1KtGrckmjIAiCAFGD6HCBqIcwMDgr8WabeHZbObQyY/\"\n - \ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": - \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": - 34,\n \"candidatesTokenCount\": 1,\n \"totalTokenCount\": 152,\n \"promptTokensDetails\": - [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 34\n - \ }\n ],\n \"thoughtsTokenCount\": 117,\n \"serviceTier\": \"standard\"\n - \ },\n \"modelVersion\": \"gemini-3.5-flash\",\n \"responseId\": \"GEwnau3jBoeHjMcP-tC5kQc\"\n}\n" - headers: - Accept-Ranges: - - none - Alt-Svc: - - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 - Content-Type: - - application/json; charset=UTF-8 - Date: - - Mon, 08 Jun 2026 23:11:21 GMT - Server: - - scaffolding on HTTPServer2 - Server-Timing: - - gfet4t7; dur=1177 - Transfer-Encoding: - - chunked - Vary: - - X-Origin - - Referer - - Origin,Accept-Encoding - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-Gemini-Service-Tier: - - standard - X-XSS-Protection: - - "0" - status: - code: 200 - message: OK +- request: + body: '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role": + "user"}], "systemInstruction": {"parts": [{"text": "Answer the user''s question + in one word.\n\nYou are an agent. Your internal name is \"my_agent\"."}], "role": + "user"}, "generationConfig": {}}' + headers: + Accept: + - '*/*' + Connection: + - keep-alive + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + accept-encoding: + - identity + x-goog-api-key: + - XXXXXX + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"text\": \"Paris\",\n \"thoughtSignature\": + \"EvADCu0DAQw51sdO/7vjnJaX3L90TuwmccG8x7ZM5JGGZirw1q95wfAmDIGBKmtDCK/EC8W/7d6UysVl1w6oNIydO7rlwwHKY4Fir3mVXwFvhWxferBmm/E1ia/nasj/OJXY7N+Cpp8D/e2GLzJWEXqYzMKEBvJ3QWvDiRpo/D3nLSuH80HF67C5Jvcb6AcXE0oPMQH2HDw4/IcmX8YTT/LWn1o3Qr6aMJvvoPq7C8J+tTpkjOkB9O4S8HOzu/u9GK7KhCk8A1GMhV1C4PQ6ger43eDqAzDf3JOiLzKzWby8pvwUL9sRMjOV4Rj4ffkGiYdeCdns5hqOBdY8Iw1HP723A9lsPVzkvVWmzLPxQrB6QXcMkApqn5zxHm5y69ar4fC90D8azUL8ckiRYGtY7LZitlpkEuul2MRljftT3WVqUc+Gi8tl/9uUp9szX2LcL+/iTGW5AKldQGFdTmOGu/yxn77ChXsZM2LaVUGWPORiSbsPE7Ln8gjvtCXk8FH1PaJH92aFfE8//V88K0QmT5VWQDHVncuhiX6oWjsPWUQem90VaKpl0F47MxuVfMKNWVkckX657xTDBNGW5+YFKqQvCvPkjvfMjEoRx+OFwJhwa7NzoAISh33QOJgR/uc9H810f3J1DuSCzfWQNREK0j/F4Q==\"\n + \ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": + \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": + 34,\n \"candidatesTokenCount\": 1,\n \"totalTokenCount\": 161,\n \"promptTokensDetails\": + [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 34\n + \ }\n ],\n \"thoughtsTokenCount\": 126,\n \"serviceTier\": \"standard\"\n + \ },\n \"modelVersion\": \"gemini-3.5-flash\",\n \"responseId\": \"ObghaqbUI9yq1MkP7qPfgQw\"\n}\n" + headers: + Accept-Ranges: + - none + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Thu, 04 Jun 2026 17:39:06 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1245 + Transfer-Encoding: + - chunked + Vary: + - X-Origin + - Referer + - Origin,Accept-Encoding + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +- request: + body: '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role": + "user"}], "systemInstruction": {"parts": [{"text": "Answer the user''s question + in one word.\n\nYou are an agent. Your internal name is \"my_agent\"."}], "role": + "user"}, "tools": [{"functionDeclarations": [{"description": "Return the capital + of a country.", "name": "get_capital", "parameters_json_schema": {"properties": + {"country": {"title": "Country", "type": "string"}}, "required": ["country"], + "title": "get_capitalParams", "type": "object"}}]}], "generationConfig": {}}' + headers: + Accept: + - '*/*' + Connection: + - keep-alive + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + accept-encoding: + - identity + x-goog-api-key: + - XXXXXX + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"functionCall\": {\n \"name\": \"get_capital\",\n + \ \"args\": {\n \"country\": \"France\"\n },\n + \ \"id\": \"9jm89vzu\"\n },\n \"thoughtSignature\": + \"ErEECq4EAQw51seB6PkS2yq4d6bzFaH2D7e0RGNPcU4OYlPM8au+0iiDtf9noT9qzch9DPIdRjEFZjUo+y+oyKymEpPyDt6/U6stHd3w64kuA0UP3qkNf7bWhLJSn+kbTxT2CB6NpBQoTIGf+yiS/tmuSj5Pda3F/xeUT6H1iktXg32jKpBhDpTV4rmy1yRM21Gi9CXRqZjrhKXksX9z6wEELMsIaQ3I9iMEmVd8+aLuz6hwa0s3/1J8MTQHAOzSOnyJjyatr6dtoCmenzOoOGBLVgkxFdg1QuAmIVCJ9oPKxVyhLbIUNt4vsEedppfGOzsWgPyegx024XvRenR48PHMDpOeFanCiGnupOfJUHJkBrmIgfFsrso0tTUCOjbHE45BZdpR8MSOr21d7QFOR2QcAvatcORmqob50UjRX5VZIUJGOwWZe3sDi+B0pFE6gN+Dmh3+EzSc9scY+o3uxjYlo6BGA5OxE0DB71ev01vxfktwbl7c97JXZy3nc2cSlWeGS8oE5YXSFwNZpEq4cjK90armVeEcRIWL/za5MwSTjQgx8xUsw1CykRmkiUI6ZTVSQxkmqEnY5HewldBvk0qILIPlQHaCOFSnDHTO1lsyw89MiHQp094O2cJxcjjuBiLdISIrdU4k+EfrG7bS92z2uAgWYJjNmV83NOhLlmGEOq4pfpvm9M+CeYbXjB65W88tBXaW2tvXHJrwR0i+7Jqa5kYZHl+P40zhFPY2FT4xu7ou\"\n + \ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": + \"STOP\",\n \"index\": 0,\n \"finishMessage\": \"Model generated + function call(s).\"\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": + 93,\n \"candidatesTokenCount\": 16,\n \"totalTokenCount\": 245,\n \"promptTokensDetails\": + [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 93\n + \ }\n ],\n \"thoughtsTokenCount\": 136,\n \"serviceTier\": \"standard\"\n + \ },\n \"modelVersion\": \"gemini-3.5-flash\",\n \"responseId\": \"Frkhasi3BoS9sOIP-dC-mQ4\"\n}\n" + headers: + Accept-Ranges: + - none + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Thu, 04 Jun 2026 17:42:47 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1410 + Transfer-Encoding: + - chunked + Vary: + - X-Origin + - Referer + - Origin,Accept-Encoding + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +- request: + body: '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role": + "user"}, {"parts": [{"functionCall": {"id": "9jm89vzu", "args": {"country": + "France"}, "name": "get_capital"}, "thoughtSignature": "ErEECq4EAQw51seB6PkS2yq4d6bzFaH2D7e0RGNPcU4OYlPM8au-0iiDtf9noT9qzch9DPIdRjEFZjUo-y-oyKymEpPyDt6_U6stHd3w64kuA0UP3qkNf7bWhLJSn-kbTxT2CB6NpBQoTIGf-yiS_tmuSj5Pda3F_xeUT6H1iktXg32jKpBhDpTV4rmy1yRM21Gi9CXRqZjrhKXksX9z6wEELMsIaQ3I9iMEmVd8-aLuz6hwa0s3_1J8MTQHAOzSOnyJjyatr6dtoCmenzOoOGBLVgkxFdg1QuAmIVCJ9oPKxVyhLbIUNt4vsEedppfGOzsWgPyegx024XvRenR48PHMDpOeFanCiGnupOfJUHJkBrmIgfFsrso0tTUCOjbHE45BZdpR8MSOr21d7QFOR2QcAvatcORmqob50UjRX5VZIUJGOwWZe3sDi-B0pFE6gN-Dmh3-EzSc9scY-o3uxjYlo6BGA5OxE0DB71ev01vxfktwbl7c97JXZy3nc2cSlWeGS8oE5YXSFwNZpEq4cjK90armVeEcRIWL_za5MwSTjQgx8xUsw1CykRmkiUI6ZTVSQxkmqEnY5HewldBvk0qILIPlQHaCOFSnDHTO1lsyw89MiHQp094O2cJxcjjuBiLdISIrdU4k-EfrG7bS92z2uAgWYJjNmV83NOhLlmGEOq4pfpvm9M-CeYbXjB65W88tBXaW2tvXHJrwR0i-7Jqa5kYZHl-P40zhFPY2FT4xu7ou"}], + "role": "model"}, {"parts": [{"functionResponse": {"id": "9jm89vzu", "name": + "get_capital", "response": {"output": "Paris"}}}], "role": "user"}], "systemInstruction": + {"parts": [{"text": "Answer the user''s question in one word.\n\nYou are an + agent. Your internal name is \"my_agent\"."}], "role": "user"}, "tools": [{"functionDeclarations": + [{"description": "Return the capital of a country.", "name": "get_capital", + "parameters_json_schema": {"properties": {"country": {"title": "Country", "type": + "string"}}, "required": ["country"], "title": "get_capitalParams", "type": "object"}}]}], + "generationConfig": {}}' + headers: + Accept: + - '*/*' + Connection: + - keep-alive + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + accept-encoding: + - identity + x-goog-api-key: + - XXXXXX + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent + response: + body: + string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": + [\n {\n \"text\": \"Paris\",\n \"thoughtSignature\": + \"Ep8BCpwBAQw51scY9jRm1gRR+wIp3n83l2Nne/45OEv3gu1A3171df1myLLVcuXK3JU59+msaJFR1qQmILPixOdxoUWnsx1CahjhgeNpocAw2WEB00xiv4G97fR3qe4w4xriy3yKxP33+kakl7o8aWFLDEqmYpr2NiYIrNjQyB54uYpuuH3jTUFGvZn64UoRsfDC1IYGPpz2zaIMpV01BsSw\"\n + \ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": + \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": + 259,\n \"candidatesTokenCount\": 1,\n \"totalTokenCount\": 286,\n \"promptTokensDetails\": + [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 259\n + \ }\n ],\n \"thoughtsTokenCount\": 26,\n \"serviceTier\": \"standard\"\n + \ },\n \"modelVersion\": \"gemini-3.5-flash\",\n \"responseId\": \"GLkhaqGsA5H6ugfVoKy4CA\"\n}\n" + headers: + Accept-Ranges: + - none + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Thu, 04 Jun 2026 17:42:48 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1122 + Transfer-Encoding: + - chunked + Vary: + - X-Origin + - Referer + - Origin,Accept-Encoding + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK version: 1 diff --git a/tests/mlmodel_googleadk/conftest.py b/tests/mlmodel_googleadk/conftest.py index 2b9db9413..467179624 100644 --- a/tests/mlmodel_googleadk/conftest.py +++ b/tests/mlmodel_googleadk/conftest.py @@ -26,6 +26,8 @@ ) from testing_support.ml_testing_utils import set_trace_info +from newrelic.common.package_version_utils import get_package_version + _default_settings = { "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow-downs. "transaction_tracer.explain_threshold": 0.0, @@ -44,8 +46,12 @@ ) +GOOGLEADK_VERSION = get_package_version("google-adk") +assert GOOGLEADK_VERSION, "Failed to pull google-adk version for supportability metric" + + @pytest.fixture(autouse=True) -def gemini_client(monkeypatch, vcr_recording): +def patch_gemini_client(monkeypatch, vcr_recording): """ Force accept-encoding: identity onto every google.genai.Client created during a test. @@ -56,6 +62,7 @@ def gemini_client(monkeypatch, vcr_recording): controlled by passing --record-mode={all, none, new_episodes} to pytest. """ + # Ensure either fake or real credentials are supplied to the Client or it won't init if vcr_recording: google_api_key = os.environ.get("GOOGLE_API_KEY") if not google_api_key: @@ -94,11 +101,11 @@ def _exercise_agent(agent, prompt, app_name="test_app", user_id="test_user"): session_service = InMemorySessionService() runner = Runner(app_name=app_name, agent=agent, session_service=session_service) - async def _run(): + async def _exercise(): session = await session_service.create_session(app_name=app_name, user_id=user_id) new_message = types.Content(role="user", parts=[types.Part.from_text(text=prompt)]) return [e async for e in runner.run_async(user_id=user_id, session_id=session.id, new_message=new_message)] - return loop.run_until_complete(_run()) + return loop.run_until_complete(_exercise()) return _exercise_agent diff --git a/tests/mlmodel_googleadk/test_tools.py b/tests/mlmodel_googleadk/test_tools.py new file mode 100644 index 000000000..90f790bfa --- /dev/null +++ b/tests/mlmodel_googleadk/test_tools.py @@ -0,0 +1,113 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _test_agent import AGENT_NAME, PROMPT, agent_recorded_event, build_agent +from _test_tools import TOOL_NAME, get_capital_tool, tool_recorded_event +from testing_support.fixtures import dt_enabled, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_with_context_attrs, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.hooks.mlmodel_googleadk import GOOGLEADK_VERSION + +EXPECTED_AGENT_METRIC = (f"Llm/agent/GoogleADK/google.adk.agents.llm_agent:LlmAgent._run_async_impl/{AGENT_NAME}", 1) +EXPECTED_TOOL_METRIC = ( + f"Llm/tool/GoogleADK/google.adk.flows.llm_flows.functions:_execute_single_function_call_async/{TOOL_NAME}", + 1, +) +EXPECTED_SUPPORTABILITY_METRIC = (f"Supportability/Python/ML/GoogleADK/{GOOGLEADK_VERSION}", 1) + + +def _validate_events(events): + assert len(events) == 3 + assert events[0].content.parts[0].function_call.name == "get_capital" + assert events[1].content.parts[0].function_response.response["output"] == "Paris" + assert events[2].content.parts[0].text == "Paris" + + +@dt_enabled +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(agent_recorded_event + tool_recorded_event(record_content=True))) +@validate_custom_event_count(count=8) # LlmAgent, LlmTool, 2x (Summary, Input, Output) for the two LLM calls. +@validate_transaction_metrics( + "test_tools:test_tool", + scoped_metrics=[EXPECTED_AGENT_METRIC, EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_AGENT_METRIC, EXPECTED_TOOL_METRIC], + custom_metrics=[EXPECTED_SUPPORTABILITY_METRIC], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": '{"type": "APM-AI_AGENT", "name": "my_agent"}'}) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool(exercise_agent, set_trace_info): + set_trace_info() + agent = build_agent(tools=[get_capital_tool]) + with WithLlmCustomAttributes({"context": "attr"}): + events = exercise_agent(agent, PROMPT) + + _validate_events(events) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(agent_recorded_event + tool_recorded_event(record_content=False)) +@validate_custom_event_count(count=8) # LlmAgent, LlmTool, 2x (Summary, Input, Output) for the two LLM calls. +@validate_transaction_metrics( + "test_tools:test_tool_no_content", + scoped_metrics=[EXPECTED_AGENT_METRIC, EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_AGENT_METRIC, EXPECTED_TOOL_METRIC], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_tool_no_content(exercise_agent, set_trace_info): + set_trace_info() + agent = build_agent(tools=[get_capital_tool]) + events = exercise_agent(agent, PROMPT) + + _validate_events(events) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@validate_transaction_metrics("test_tools:test_tool_disabled_ai_monitoring", background_task=True) +@background_task() +def test_tool_disabled_ai_monitoring(exercise_agent, set_trace_info): + set_trace_info() + agent = build_agent(tools=[get_capital_tool]) + events = exercise_agent(agent, PROMPT) + + _validate_events(events) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_tool_outside_transaction(exercise_agent, set_trace_info): + set_trace_info() + agent = build_agent(tools=[get_capital_tool]) + events = exercise_agent(agent, PROMPT) + + _validate_events(events) diff --git a/tests/mlmodel_googleadk/test_tools_error.py b/tests/mlmodel_googleadk/test_tools_error.py new file mode 100644 index 000000000..be7242525 --- /dev/null +++ b/tests/mlmodel_googleadk/test_tools_error.py @@ -0,0 +1,56 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from _test_agent import AGENT_NAME, PROMPT, agent_recorded_event_error, build_agent +from _test_tools import TOOL_NAME, raising_tool, tool_recorded_event_error +from testing_support.fixtures import dt_enabled, reset_core_stats_engine, validate_attributes +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_error_event_count import validate_transaction_error_event_count +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.common.object_names import callable_name + +EXPECTED_AGENT_METRIC = (f"Llm/agent/GoogleADK/google.adk.agents.llm_agent:LlmAgent._run_async_impl/{AGENT_NAME}", 1) +EXPECTED_TOOL_METRIC = ( + f"Llm/tool/GoogleADK/google.adk.flows.llm_flows.functions:_execute_single_function_call_async/{TOOL_NAME}", + 1, +) + + +@dt_enabled +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(ValueError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(agent_recorded_event_error + tool_recorded_event_error(record_content=True)) +@validate_custom_event_count(count=5) # LlmAgent, LlmTool, Summary, Input, Output +@validate_transaction_metrics( + "test_tools_error:test_tool_error", + scoped_metrics=[EXPECTED_AGENT_METRIC, EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_AGENT_METRIC, EXPECTED_TOOL_METRIC], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": '{"type": "APM-AI_AGENT", "name": "my_agent"}'}) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool_error(exercise_agent, set_trace_info): + set_trace_info() + agent = build_agent(tools=[raising_tool]) + with pytest.raises(ValueError): + exercise_agent(agent, PROMPT) From eb53c88aa370d3ec4bc202a45096b21c974ecc30 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Tue, 16 Jun 2026 12:04:38 -0700 Subject: [PATCH 2/5] Expand Megalinter ignored files --- .mega-linter.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.mega-linter.yml b/.mega-linter.yml index ef6d98461..f700832cf 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -25,7 +25,15 @@ CLEAR_REPORT_FOLDER: true VALIDATE_ALL_CODEBASE: true IGNORE_GITIGNORED_FILES: true FAIL_IF_MISSING_LINTER_IN_FLAVOR: true -FILTER_REGEX_EXCLUDE: "(.*/?packages/.*)" # Ignore packages directories +FILTER_REGEX_EXCLUDE: "(.*cassette.*\\.ya?ml)" # Ignore recorded cassette.yaml files +ADDITIONAL_EXCLUDED_DIRECTORIES: + - .pytest_cache + - .ruff_cache + - .tox + - .venv + - megalinter-reports + - newrelic.egg-info + - newrelic/packages ENABLE_LINTERS: # If you use ENABLE_LINTERS variable, all other linters will be disabled by default - ACTION_ACTIONLINT From 9e66ee0fd1864a424e4c9bd430ae4c1e27d38dab Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Wed, 24 Jun 2026 12:49:13 -0700 Subject: [PATCH 3/5] Apply suggestions from code review --- newrelic/hooks/mlmodel_googleadk.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/newrelic/hooks/mlmodel_googleadk.py b/newrelic/hooks/mlmodel_googleadk.py index 54db81112..5884d5b9d 100644 --- a/newrelic/hooks/mlmodel_googleadk.py +++ b/newrelic/hooks/mlmodel_googleadk.py @@ -90,8 +90,9 @@ async def wrap__execute_single_function_call_async(wrapped, instance, args, kwar transaction=transaction, linking_metadata=linking_metadata, ) - tool_event_dict["duration"] = ft.duration * 1000 - transaction.record_custom_event("LlmTool", tool_event_dict) + if tool_event_dict: + tool_event_dict["duration"] = ft.duration * 1000 + transaction.record_custom_event("LlmTool", tool_event_dict) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) raise @@ -110,8 +111,9 @@ async def wrap__execute_single_function_call_async(wrapped, instance, args, kwar transaction=transaction, linking_metadata=linking_metadata, ) - tool_event_dict["duration"] = ft.duration * 1000 - transaction.record_custom_event("LlmTool", tool_event_dict) + if tool_event_dict: + tool_event_dict["duration"] = ft.duration * 1000 + transaction.record_custom_event("LlmTool", tool_event_dict) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) @@ -152,7 +154,7 @@ def _construct_base_tool_event_dict( if settings.ai_monitoring.record_content.enabled: tool_event_dict["input"] = str(tool_input) if tool_input else None - tool_event_dict["output"] = str(tool_output) if tool_output is not None else None + tool_event_dict["output"] = str(tool_output) if tool_output else None tool_event_dict.update(_get_llm_metadata(transaction)) except Exception: From 0088aeaae51ab4d4dbd3398efb14302837964c30 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Wed, 24 Jun 2026 14:44:27 -0700 Subject: [PATCH 4/5] Gate subcomponent attrs to local tools --- newrelic/hooks/mlmodel_googleadk.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/newrelic/hooks/mlmodel_googleadk.py b/newrelic/hooks/mlmodel_googleadk.py index 5884d5b9d..867a4e26e 100644 --- a/newrelic/hooks/mlmodel_googleadk.py +++ b/newrelic/hooks/mlmodel_googleadk.py @@ -50,14 +50,20 @@ async def wrap__execute_single_function_call_async(wrapped, instance, args, kwar run_id = "" tool_input = None agent_name = "agent" + is_local_tool = False try: bound_args = bind_args(wrapped, args, kwargs) function_call = bound_args.get("function_call") agent = bound_args.get("agent") + tools_dict = bound_args.get("tools_dict") if function_call is not None: tool_name = getattr(function_call, "name", "tool") or "tool" run_id = getattr(function_call, "id", "") or "" tool_input = getattr(function_call, "args", None) + if tools_dict is not None: + from google.adk.tools.function_tool import FunctionTool + + is_local_tool = isinstance(tools_dict.get(tool_name), FunctionTool) if agent is not None: agent_name = getattr(agent, "name", "agent") or "agent" except Exception: @@ -65,11 +71,12 @@ async def wrap__execute_single_function_call_async(wrapped, instance, args, kwar func_name = callable_name(wrapped) function_trace_name = f"{func_name}/{tool_name}" - agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} ft = FunctionTrace(name=function_trace_name, group="Llm/tool/GoogleADK") ft.__enter__() - ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + if is_local_tool: + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) linking_metadata = get_trace_linking_metadata() tool_id = str(uuid.uuid4()) From bf0047793dfe8a1e001517e2a56c797c7d574940 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Wed, 24 Jun 2026 14:57:12 -0700 Subject: [PATCH 5/5] Renaming according to code review --- newrelic/hooks/mlmodel_googleadk.py | 4 +--- tests/mlmodel_googleadk/test_tools.py | 7 ++----- tests/mlmodel_googleadk/test_tools_error.py | 7 ++----- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/newrelic/hooks/mlmodel_googleadk.py b/newrelic/hooks/mlmodel_googleadk.py index 29abbb0d7..eaeae49a0 100644 --- a/newrelic/hooks/mlmodel_googleadk.py +++ b/newrelic/hooks/mlmodel_googleadk.py @@ -21,7 +21,6 @@ from newrelic.api.time_trace import get_trace_linking_metadata from newrelic.api.transaction import current_transaction from newrelic.common.llm_utils import AsyncLLMStreamProxy, _get_llm_metadata -from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version from newrelic.common.signature import bind_args @@ -183,8 +182,7 @@ async def wrap__execute_single_function_call_async(wrapped, instance, args, kwar except Exception: _logger.warning(TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) - func_name = callable_name(wrapped) - function_trace_name = f"{func_name}/{tool_name}" + function_trace_name = f"execute_single_function_call_async/{tool_name}" ft = FunctionTrace(name=function_trace_name, group="Llm/tool/GoogleADK") ft.__enter__() diff --git a/tests/mlmodel_googleadk/test_tools.py b/tests/mlmodel_googleadk/test_tools.py index 90f790bfa..07edfa899 100644 --- a/tests/mlmodel_googleadk/test_tools.py +++ b/tests/mlmodel_googleadk/test_tools.py @@ -29,11 +29,8 @@ from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes from newrelic.hooks.mlmodel_googleadk import GOOGLEADK_VERSION -EXPECTED_AGENT_METRIC = (f"Llm/agent/GoogleADK/google.adk.agents.llm_agent:LlmAgent._run_async_impl/{AGENT_NAME}", 1) -EXPECTED_TOOL_METRIC = ( - f"Llm/tool/GoogleADK/google.adk.flows.llm_flows.functions:_execute_single_function_call_async/{TOOL_NAME}", - 1, -) +EXPECTED_AGENT_METRIC = (f"Llm/agent/GoogleADK/run_async/{AGENT_NAME}", 1) +EXPECTED_TOOL_METRIC = (f"Llm/tool/GoogleADK/execute_single_function_call_async/{TOOL_NAME}", 1) EXPECTED_SUPPORTABILITY_METRIC = (f"Supportability/Python/ML/GoogleADK/{GOOGLEADK_VERSION}", 1) diff --git a/tests/mlmodel_googleadk/test_tools_error.py b/tests/mlmodel_googleadk/test_tools_error.py index be7242525..510a5e201 100644 --- a/tests/mlmodel_googleadk/test_tools_error.py +++ b/tests/mlmodel_googleadk/test_tools_error.py @@ -26,11 +26,8 @@ from newrelic.api.background_task import background_task from newrelic.common.object_names import callable_name -EXPECTED_AGENT_METRIC = (f"Llm/agent/GoogleADK/google.adk.agents.llm_agent:LlmAgent._run_async_impl/{AGENT_NAME}", 1) -EXPECTED_TOOL_METRIC = ( - f"Llm/tool/GoogleADK/google.adk.flows.llm_flows.functions:_execute_single_function_call_async/{TOOL_NAME}", - 1, -) +EXPECTED_AGENT_METRIC = (f"Llm/agent/GoogleADK/run_async/{AGENT_NAME}", 1) +EXPECTED_TOOL_METRIC = (f"Llm/tool/GoogleADK/execute_single_function_call_async/{TOOL_NAME}", 1) @dt_enabled