Skip to content

Commit 173e87a

Browse files
caohy1988claude
andcommitted
feat(plugins): add BQAA tests and view columns for error events
Follow-up to google#5045. Adds BQ plugin-specific test coverage and enriches the analytics views for AGENT_ERROR and INVOCATION_ERROR events. Changes: - Add error_traceback column extraction to v_agent_error and v_invocation_error views (JSON_VALUE from content field) - Add test_on_agent_error_callback_logs_correctly: verifies AGENT_ERROR event is logged with correct error_message, status, and traceback - Add test_on_run_error_callback_logs_correctly: verifies INVOCATION_ERROR event is logged with correct fields - Add TestRunErrorCallbackCleanupSafety: verifies TraceManager cleanup runs even when _log_event raises during on_run_error_callback - Add test_error_views_contain_traceback_column: verifies view SQL includes error_traceback extraction Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 8a8b3a2 commit 173e87a

File tree

2 files changed

+151
-1
lines changed

2 files changed

+151
-1
lines changed

src/google/adk/plugins/bigquery_agent_analytics_plugin.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1766,10 +1766,13 @@ def _get_events_schema() -> list[bigquery.SchemaField]:
17661766
],
17671767
"AGENT_ERROR": [
17681768
"CAST(JSON_VALUE(latency_ms, '$.total_ms') AS INT64) AS total_ms",
1769+
"JSON_VALUE(content, '$.error_traceback') AS error_traceback",
17691770
],
17701771
"INVOCATION_STARTING": [],
17711772
"INVOCATION_COMPLETED": [],
1772-
"INVOCATION_ERROR": [],
1773+
"INVOCATION_ERROR": [
1774+
"JSON_VALUE(content, '$.error_traceback') AS error_traceback",
1775+
],
17731776
"STATE_DELTA": [
17741777
"JSON_QUERY(attributes, '$.state_delta') AS state_delta",
17751778
],

tests/unittests/plugins/test_bigquery_agent_analytics_plugin.py

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1765,6 +1765,63 @@ async def test_on_tool_error_callback_logs_correctly(
17651765
assert log_entry["error_message"] == "Tool timed out"
17661766
assert log_entry["status"] == "ERROR"
17671767

1768+
@pytest.mark.asyncio
1769+
async def test_on_agent_error_callback_logs_correctly(
1770+
self,
1771+
bq_plugin_inst,
1772+
mock_write_client,
1773+
mock_agent,
1774+
callback_context,
1775+
dummy_arrow_schema,
1776+
):
1777+
error = RuntimeError("Agent crashed")
1778+
bigquery_agent_analytics_plugin.TraceManager.push_span(
1779+
callback_context, "agent"
1780+
)
1781+
await bq_plugin_inst.on_agent_error_callback(
1782+
agent=mock_agent,
1783+
callback_context=callback_context,
1784+
error=error,
1785+
)
1786+
await asyncio.sleep(0.01)
1787+
log_entry = await _get_captured_event_dict_async(
1788+
mock_write_client, dummy_arrow_schema
1789+
)
1790+
_assert_common_fields(log_entry, "AGENT_ERROR")
1791+
assert log_entry["error_message"] == "Agent crashed"
1792+
assert log_entry["status"] == "ERROR"
1793+
content_dict = json.loads(log_entry["content"])
1794+
assert "error_traceback" in content_dict
1795+
assert "RuntimeError: Agent crashed" in content_dict["error_traceback"]
1796+
1797+
@pytest.mark.asyncio
1798+
async def test_on_run_error_callback_logs_correctly(
1799+
self,
1800+
bq_plugin_inst,
1801+
mock_write_client,
1802+
invocation_context,
1803+
callback_context,
1804+
dummy_arrow_schema,
1805+
):
1806+
error = ValueError("Invocation failed")
1807+
bigquery_agent_analytics_plugin.TraceManager.push_span(
1808+
callback_context, "invocation"
1809+
)
1810+
await bq_plugin_inst.on_run_error_callback(
1811+
invocation_context=invocation_context,
1812+
error=error,
1813+
)
1814+
await asyncio.sleep(0.01)
1815+
log_entry = await _get_captured_event_dict_async(
1816+
mock_write_client, dummy_arrow_schema
1817+
)
1818+
_assert_common_fields(log_entry, "INVOCATION_ERROR")
1819+
assert log_entry["error_message"] == "Invocation failed"
1820+
assert log_entry["status"] == "ERROR"
1821+
content_dict = json.loads(log_entry["content"])
1822+
assert "error_traceback" in content_dict
1823+
assert "ValueError: Invocation failed" in content_dict["error_traceback"]
1824+
17681825
@pytest.mark.asyncio
17691826
async def test_table_creation_options(
17701827
self,
@@ -5147,6 +5204,31 @@ def test_view_sql_contains_correct_event_filter(self):
51475204
view_name = "v_" + event_type.lower()
51485205
assert view_name in all_sql, f"View {view_name} not found in SQL"
51495206

5207+
def test_error_views_contain_traceback_column(self):
5208+
"""AGENT_ERROR and INVOCATION_ERROR views extract error_traceback."""
5209+
plugin = self._make_plugin(create_views=True)
5210+
plugin.client.get_table.side_effect = cloud_exceptions.NotFound("not found")
5211+
mock_query_job = mock.MagicMock()
5212+
plugin.client.query.return_value = mock_query_job
5213+
5214+
plugin._ensure_schema_exists()
5215+
5216+
calls = plugin.client.query.call_args_list
5217+
view_sqls = {c[0][0]: c[0][0] for c in calls}
5218+
5219+
# Find the AGENT_ERROR and INVOCATION_ERROR view SQLs.
5220+
agent_error_sqls = [sql for sql in view_sqls if "v_agent_error" in sql]
5221+
invocation_error_sqls = [
5222+
sql for sql in view_sqls if "v_invocation_error" in sql
5223+
]
5224+
5225+
assert len(agent_error_sqls) == 1
5226+
assert "error_traceback" in agent_error_sqls[0]
5227+
assert "total_ms" in agent_error_sqls[0]
5228+
5229+
assert len(invocation_error_sqls) == 1
5230+
assert "error_traceback" in invocation_error_sqls[0]
5231+
51505232
def test_config_create_views_default_true(self):
51515233
"""Config create_views defaults to True."""
51525234
config = bigquery_agent_analytics_plugin.BigQueryLoggerConfig()
@@ -6187,6 +6269,71 @@ async def test_cleanup_runs_when_log_event_raises(
61876269
provider.shutdown()
61886270

61896271

6272+
class TestRunErrorCallbackCleanupSafety:
6273+
"""on_run_error_callback cleanup must execute even if _log_event fails."""
6274+
6275+
@pytest.mark.asyncio
6276+
async def test_cleanup_runs_when_log_event_raises(
6277+
self,
6278+
bq_plugin_inst,
6279+
mock_write_client,
6280+
invocation_context,
6281+
callback_context,
6282+
mock_agent,
6283+
):
6284+
"""Stale state is cleared even when _log_event raises in error cb."""
6285+
from opentelemetry.sdk.trace import TracerProvider as SdkProvider
6286+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
6287+
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
6288+
6289+
provider = SdkProvider()
6290+
provider.add_span_processor(SimpleSpanProcessor(InMemorySpanExporter()))
6291+
real_tracer = provider.get_tracer("test")
6292+
6293+
with mock.patch.object(
6294+
bigquery_agent_analytics_plugin, "tracer", real_tracer
6295+
):
6296+
bigquery_agent_analytics_plugin._span_records_ctx.set(None)
6297+
bigquery_agent_analytics_plugin._active_invocation_id_ctx.set(None)
6298+
bigquery_agent_analytics_plugin._root_agent_name_ctx.set(None)
6299+
6300+
# Run before_run to initialise state.
6301+
await bq_plugin_inst.before_run_callback(
6302+
invocation_context=invocation_context
6303+
)
6304+
6305+
# Verify state is populated.
6306+
assert bigquery_agent_analytics_plugin._span_records_ctx.get()
6307+
assert (
6308+
bigquery_agent_analytics_plugin._active_invocation_id_ctx.get()
6309+
is not None
6310+
)
6311+
6312+
# Make _log_event raise inside on_run_error_callback.
6313+
with mock.patch.object(
6314+
bq_plugin_inst,
6315+
"_log_event",
6316+
side_effect=RuntimeError("boom"),
6317+
):
6318+
# _safe_callback swallows the exception, but cleanup in
6319+
# the finally block must still execute.
6320+
await bq_plugin_inst.on_run_error_callback(
6321+
invocation_context=invocation_context,
6322+
error=ValueError("crash"),
6323+
)
6324+
6325+
# All invocation state must be cleaned up despite the error.
6326+
records = bigquery_agent_analytics_plugin._span_records_ctx.get()
6327+
assert records == [] or records is None
6328+
assert (
6329+
bigquery_agent_analytics_plugin._active_invocation_id_ctx.get()
6330+
is None
6331+
)
6332+
assert bigquery_agent_analytics_plugin._root_agent_name_ctx.get() is None
6333+
6334+
provider.shutdown()
6335+
6336+
61906337
class TestStringSystemPromptTruncation:
61916338
"""Tests that a string system prompt is truncated in parse()."""
61926339

0 commit comments

Comments
 (0)