Skip to content

Commit 7e9345b

Browse files
RKestcopybara-github
RKest
authored and
copybara-github
committed
test(telemetry): assert metric points in telemetry functional tests
Add a `metrics` component to FunctionalTestCase so the parametrized telemetry functional tests assert recorded metric points alongside spans and logs. Relocate the in-memory metric reader setup + the MetricPoint digest into functional_test_helpers so both test_functional.py and test_node_functional.py share them. Delete the standalone metrics-only functional tests (test_metrics, test_metrics_tool_error); their coverage is now subsumed by the per-case metric expectations. Co-authored-by: Max Ind <maxind@google.com> PiperOrigin-RevId: 939790021
1 parent c84c17a commit 7e9345b

5 files changed

Lines changed: 360 additions & 322 deletions

File tree

‎tests/unittests/telemetry/functional_node_test_cases.py‎

Lines changed: 76 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,18 @@
4242
from .functional_test_helpers import GEN_AI_SYSTEM_MESSAGE_EVENT
4343
from .functional_test_helpers import GEN_AI_USER_MESSAGE_EVENT
4444
from .functional_test_helpers import LogDigest
45+
from .functional_test_helpers import MetricPoint
4546
from .functional_test_helpers import NODE_NAME
4647
from .functional_test_helpers import NODE_RESULT
48+
from .functional_test_helpers import NON_DETERMINISTIC
4749
from .functional_test_helpers import PRESENT
4850
from .functional_test_helpers import SpanDigest
51+
from .functional_test_helpers import TelemetryDigest
4952
from .functional_test_helpers import TOOL_ARGS
5053
from .functional_test_helpers import TOOL_DESCRIPTION
5154
from .functional_test_helpers import TOOL_NAME
5255
from .functional_test_helpers import TOOL_RESULT
56+
from .functional_test_helpers import USER_PROMPT
5357
from .functional_test_helpers import WORKFLOW_NAME
5458

5559
# The agent's "user" input in this scenario is the node's output, since
@@ -1503,6 +1507,54 @@
15031507
)
15041508

15051509

1510+
# Expected metric points, grouped by metric name.
1511+
EXPECTED_NODE_METRICS: dict[str, frozenset[MetricPoint]] = {
1512+
"gen_ai.agent.invocation.duration": frozenset({
1513+
MetricPoint(
1514+
attributes={"gen_ai.agent.name": AGENT_NAME},
1515+
value=NON_DETERMINISTIC,
1516+
),
1517+
}),
1518+
"gen_ai.tool.execution.duration": frozenset({
1519+
MetricPoint(
1520+
attributes={
1521+
"gen_ai.agent.name": AGENT_NAME,
1522+
"gen_ai.tool.name": TOOL_NAME,
1523+
"gen_ai.tool.type": "FunctionTool",
1524+
},
1525+
value=NON_DETERMINISTIC,
1526+
),
1527+
}),
1528+
"gen_ai.agent.request.size": frozenset({
1529+
MetricPoint(
1530+
attributes={"gen_ai.agent.name": AGENT_NAME},
1531+
value=len(USER_PROMPT),
1532+
),
1533+
}),
1534+
"gen_ai.agent.response.size": frozenset({
1535+
MetricPoint(
1536+
attributes={"gen_ai.agent.name": AGENT_NAME},
1537+
value=len(FINAL_TEXT),
1538+
),
1539+
}),
1540+
"gen_ai.agent.workflow.steps": frozenset({
1541+
MetricPoint(attributes={"gen_ai.agent.name": AGENT_NAME}, value=3),
1542+
}),
1543+
"gen_ai.client.operation.duration": frozenset({
1544+
MetricPoint(
1545+
attributes={
1546+
"gen_ai.agent.name": AGENT_NAME,
1547+
"gen_ai.operation.name": "generate_content",
1548+
"gen_ai.provider.name": "gemini",
1549+
"gen_ai.request.model": "mock",
1550+
"gen_ai.response.model": "mock",
1551+
},
1552+
value=NON_DETERMINISTIC,
1553+
),
1554+
}),
1555+
}
1556+
1557+
15061558
# ---------------------------------------------------------------------------
15071559
# Parametrization list.
15081560
# ---------------------------------------------------------------------------
@@ -1512,36 +1564,54 @@
15121564
test_id="stable-no-capture",
15131565
semconv_opt_in=None,
15141566
capture_content="false",
1515-
expected_root=EXPECTED_STABLE_NO_CAPTURE,
1567+
expected=TelemetryDigest(
1568+
root_span=EXPECTED_STABLE_NO_CAPTURE,
1569+
metric_points=EXPECTED_NODE_METRICS,
1570+
),
15161571
),
15171572
FunctionalTestCase(
15181573
test_id="stable-capture",
15191574
semconv_opt_in=None,
15201575
capture_content="true",
1521-
expected_root=EXPECTED_STABLE_CAPTURE,
1576+
expected=TelemetryDigest(
1577+
root_span=EXPECTED_STABLE_CAPTURE,
1578+
metric_points=EXPECTED_NODE_METRICS,
1579+
),
15221580
),
15231581
FunctionalTestCase(
15241582
test_id="experimental-no-content",
15251583
semconv_opt_in=EXPERIMENTAL_OPT_IN,
15261584
capture_content="no_content",
1527-
expected_root=EXPECTED_EXPERIMENTAL_NO_CONTENT,
1585+
expected=TelemetryDigest(
1586+
root_span=EXPECTED_EXPERIMENTAL_NO_CONTENT,
1587+
metric_points=EXPECTED_NODE_METRICS,
1588+
),
15281589
),
15291590
FunctionalTestCase(
15301591
test_id="experimental-span-only",
15311592
semconv_opt_in=EXPERIMENTAL_OPT_IN,
15321593
capture_content="span_only",
1533-
expected_root=EXPECTED_EXPERIMENTAL_SPAN_ONLY,
1594+
expected=TelemetryDigest(
1595+
root_span=EXPECTED_EXPERIMENTAL_SPAN_ONLY,
1596+
metric_points=EXPECTED_NODE_METRICS,
1597+
),
15341598
),
15351599
FunctionalTestCase(
15361600
test_id="experimental-event-only",
15371601
semconv_opt_in=EXPERIMENTAL_OPT_IN,
15381602
capture_content="event_only",
1539-
expected_root=EXPECTED_EXPERIMENTAL_EVENT_ONLY,
1603+
expected=TelemetryDigest(
1604+
root_span=EXPECTED_EXPERIMENTAL_EVENT_ONLY,
1605+
metric_points=EXPECTED_NODE_METRICS,
1606+
),
15401607
),
15411608
FunctionalTestCase(
15421609
test_id="experimental-span-and-event",
15431610
semconv_opt_in=EXPERIMENTAL_OPT_IN,
15441611
capture_content="span_and_event",
1545-
expected_root=EXPECTED_EXPERIMENTAL_SPAN_AND_EVENT,
1612+
expected=TelemetryDigest(
1613+
root_span=EXPECTED_EXPERIMENTAL_SPAN_AND_EVENT,
1614+
metric_points=EXPECTED_NODE_METRICS,
1615+
),
15461616
),
15471617
]

‎tests/unittests/telemetry/functional_test_cases.py‎

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,11 @@
4040
from .functional_test_helpers import GEN_AI_SYSTEM_MESSAGE_EVENT
4141
from .functional_test_helpers import GEN_AI_USER_MESSAGE_EVENT
4242
from .functional_test_helpers import LogDigest
43+
from .functional_test_helpers import MetricPoint
44+
from .functional_test_helpers import NON_DETERMINISTIC
4345
from .functional_test_helpers import PRESENT
4446
from .functional_test_helpers import SpanDigest
47+
from .functional_test_helpers import TelemetryDigest
4548
from .functional_test_helpers import TOOL_ARGS
4649
from .functional_test_helpers import TOOL_DESCRIPTION
4750
from .functional_test_helpers import TOOL_NAME
@@ -1266,6 +1269,54 @@
12661269
)
12671270

12681271

1272+
# Expected metric points, grouped by metric name.
1273+
EXPECTED_METRICS: dict[str, frozenset[MetricPoint]] = {
1274+
"gen_ai.agent.invocation.duration": frozenset({
1275+
MetricPoint(
1276+
attributes={"gen_ai.agent.name": AGENT_NAME},
1277+
value=NON_DETERMINISTIC,
1278+
),
1279+
}),
1280+
"gen_ai.tool.execution.duration": frozenset({
1281+
MetricPoint(
1282+
attributes={
1283+
"gen_ai.agent.name": AGENT_NAME,
1284+
"gen_ai.tool.name": TOOL_NAME,
1285+
"gen_ai.tool.type": "FunctionTool",
1286+
},
1287+
value=NON_DETERMINISTIC,
1288+
),
1289+
}),
1290+
"gen_ai.agent.request.size": frozenset({
1291+
MetricPoint(
1292+
attributes={"gen_ai.agent.name": AGENT_NAME},
1293+
value=len(USER_PROMPT),
1294+
),
1295+
}),
1296+
"gen_ai.agent.response.size": frozenset({
1297+
MetricPoint(
1298+
attributes={"gen_ai.agent.name": AGENT_NAME},
1299+
value=len(FINAL_TEXT),
1300+
),
1301+
}),
1302+
"gen_ai.agent.workflow.steps": frozenset({
1303+
MetricPoint(attributes={"gen_ai.agent.name": AGENT_NAME}, value=3),
1304+
}),
1305+
"gen_ai.client.operation.duration": frozenset({
1306+
MetricPoint(
1307+
attributes={
1308+
"gen_ai.agent.name": AGENT_NAME,
1309+
"gen_ai.operation.name": "generate_content",
1310+
"gen_ai.provider.name": "gemini",
1311+
"gen_ai.request.model": "mock",
1312+
"gen_ai.response.model": "mock",
1313+
},
1314+
value=NON_DETERMINISTIC,
1315+
),
1316+
}),
1317+
}
1318+
1319+
12691320
# ---------------------------------------------------------------------------
12701321
# Parametrization list.
12711322
# ---------------------------------------------------------------------------
@@ -1275,36 +1326,54 @@
12751326
test_id="stable-no-capture",
12761327
semconv_opt_in=None,
12771328
capture_content="false",
1278-
expected_root=EXPECTED_STABLE_NO_CAPTURE,
1329+
expected=TelemetryDigest(
1330+
root_span=EXPECTED_STABLE_NO_CAPTURE,
1331+
metric_points=EXPECTED_METRICS,
1332+
),
12791333
),
12801334
FunctionalTestCase(
12811335
test_id="stable-capture",
12821336
semconv_opt_in=None,
12831337
capture_content="true",
1284-
expected_root=EXPECTED_STABLE_CAPTURE,
1338+
expected=TelemetryDigest(
1339+
root_span=EXPECTED_STABLE_CAPTURE,
1340+
metric_points=EXPECTED_METRICS,
1341+
),
12851342
),
12861343
FunctionalTestCase(
12871344
test_id="experimental-no-content",
12881345
semconv_opt_in=EXPERIMENTAL_OPT_IN,
12891346
capture_content="no_content",
1290-
expected_root=EXPECTED_EXPERIMENTAL_NO_CONTENT,
1347+
expected=TelemetryDigest(
1348+
root_span=EXPECTED_EXPERIMENTAL_NO_CONTENT,
1349+
metric_points=EXPECTED_METRICS,
1350+
),
12911351
),
12921352
FunctionalTestCase(
12931353
test_id="experimental-span-only",
12941354
semconv_opt_in=EXPERIMENTAL_OPT_IN,
12951355
capture_content="span_only",
1296-
expected_root=EXPECTED_EXPERIMENTAL_SPAN_ONLY,
1356+
expected=TelemetryDigest(
1357+
root_span=EXPECTED_EXPERIMENTAL_SPAN_ONLY,
1358+
metric_points=EXPECTED_METRICS,
1359+
),
12971360
),
12981361
FunctionalTestCase(
12991362
test_id="experimental-event-only",
13001363
semconv_opt_in=EXPERIMENTAL_OPT_IN,
13011364
capture_content="event_only",
1302-
expected_root=EXPECTED_EXPERIMENTAL_EVENT_ONLY,
1365+
expected=TelemetryDigest(
1366+
root_span=EXPECTED_EXPERIMENTAL_EVENT_ONLY,
1367+
metric_points=EXPECTED_METRICS,
1368+
),
13031369
),
13041370
FunctionalTestCase(
13051371
test_id="experimental-span-and-event",
13061372
semconv_opt_in=EXPERIMENTAL_OPT_IN,
13071373
capture_content="span_and_event",
1308-
expected_root=EXPECTED_EXPERIMENTAL_SPAN_AND_EVENT,
1374+
expected=TelemetryDigest(
1375+
root_span=EXPECTED_EXPERIMENTAL_SPAN_AND_EVENT,
1376+
metric_points=EXPECTED_METRICS,
1377+
),
13091378
),
13101379
]

0 commit comments

Comments
 (0)