Skip to content

Simplify validation in conformance tests #276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lmolkova merged 3 commits into from
Jul 16, 2026
Merged

Simplify validation in conformance tests #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
99fe694
Make conformance expected_spans an exact per-operation count map
lmolkova Jul 16, 2026
09cafc3
Potential fix for pull request finding
lmolkova Jul 16, 2026
41b8f96
Merge remote-tracking branch 'origin/main' into refactor/conformance-…
lmolkova Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions .github/skills/write-conformance-tests/SKILL.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,12 @@ Each scenario module defines a subclass of `Scenario` from
`opentelemetry.test_util_genai.conformance`. It sets the `expected_spans` /
`expected_metrics` ClassVars and implements
`run(self, *, tracer_provider, meter_provider, logger_provider, vcr)`.

`expected_spans` is a `dict[str, int]` of exact per-operation span counts
(e.g. `{"chat": 2}` for a tool-calling turn's two LLM calls); the base
`validate` fails on any mismatch. Pin real counts here — don't re-assert them
in a `validate` override. `expected_metrics` is a tuple of metric names that
must each appear at least once.
Drive instrumentation through the shared `instrument` context manager (not
`instr.instrument()` / `trace.set_tracer_provider`). The runner injects an
already-configured `vcr`, so a cassette-based scenario just calls
Expand All @@ -121,7 +127,7 @@ from opentelemetry.test_util_genai.instrumentor import instrument


class InferenceScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down Expand Up @@ -219,7 +225,7 @@ from opentelemetry.test_util_genai.conformance import Scenario


class ToolCallingScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 2} # initial turn + follow-up with tool results
expected_metrics = ("gen_ai.client.operation.duration",)

def run(self, *, tracer_provider, meter_provider, logger_provider, vcr) -> None:
Expand Down
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@


class InferenceScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
2 changes: 1 addition & 1 deletion ...mentation/opentelemetry-instrumentation-genai-anthropic/tests/conformance/tool_calling.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@


class ToolCallingScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
20 changes: 1 addition & 19 deletions instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.test.weaver_live_check import LiveCheckReport
from opentelemetry.test_util_genai.conformance import (
ExpectedViolation,
Scenario,
Expand All @@ -39,7 +38,7 @@ def add(a: float, b: float) -> float:


class AgentScenario(Scenario):
expected_spans = ("invoke_agent", "chat", "chat")
expected_spans = {"invoke_agent": 1, "chat": 3, "execute_tool": 2}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down Expand Up @@ -100,20 +99,3 @@ def run(
]
}
)

def validate(self, report: LiveCheckReport) -> None:
super().validate(report)
operations = [
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.operation.name"
]
assert "invoke_agent" in operations, (
f"Expected an invoke_agent span; saw operations {operations}"
)
assert operations.count("chat") >= 2, (
"ReAct agent exercises at least two chat completions "
f"(initial tool-call request and follow-up); saw {operations}"
)
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@


class InferenceScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
22 changes: 1 addition & 21 deletions ...mentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/tool_calling.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.test.weaver_live_check import LiveCheckReport
from opentelemetry.test_util_genai.conformance import (
ExpectedViolation,
Scenario,
Expand Down Expand Up @@ -71,7 +70,7 @@ def _execute_weather_tool(arguments: str) -> str:


class ToolCallingScenario(Scenario):
expected_spans = ("chat", "execute_tool")
expected_spans = {"chat": 2, "execute_tool": 2}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down Expand Up @@ -142,22 +141,3 @@ def run(
)

llm_with_tools.invoke(messages)

def validate(self, report: LiveCheckReport) -> None:
super().validate(report)
operations = [
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.operation.name"
]
assert operations == [
"chat",
"execute_tool",
"execute_tool",
"chat",
], (
"Tool calling exercises two chat completions with two execute_tool "
"spans in between (one per tool call); saw spans {operations}"
)
20 changes: 1 addition & 19 deletions instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.test.weaver_live_check import LiveCheckReport
from opentelemetry.test_util_genai.conformance import (
ExpectedViolation,
Scenario,
Expand All @@ -32,7 +31,7 @@ class GraphState(TypedDict):


class WorkflowScenario(Scenario):
expected_spans = ("invoke_workflow", "chat", "chat")
expected_spans = {"invoke_workflow": 1, "chat": 2}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down Expand Up @@ -119,20 +118,3 @@ def summariser(state: GraphState) -> dict:
"research": "",
}
)

def validate(self, report: LiveCheckReport) -> None:
super().validate(report)
operations = [
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.operation.name"
]
assert "invoke_workflow" in operations, (
f"Expected an invoke_workflow span; saw operations {operations}"
)
assert operations.count("chat") >= 2, (
"Two-node workflow exercises two chat completions "
f"(researcher and summariser); saw {operations}"
)
25 changes: 5 additions & 20 deletions ...tion/opentelemetry-instrumentation-genai-openai-agents/tests/conformance/orchestration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,11 @@ def _build_triage_agent() -> Agent:


class OrchestrationScenario(Scenario):
expected_spans = (
"invoke_workflow",
"invoke_agent",
"execute_tool",
)
expected_spans = {
"invoke_workflow": 1,
"invoke_agent": 2,
"execute_tool": 1,
}
expected_metrics = ("gen_ai.client.operation.duration",)
expected_violations = (
# `FunctionSpanData` in the openai-agents library doesn't expose
Expand Down Expand Up @@ -120,28 +120,13 @@ def run(

def validate(self, report: LiveCheckReport) -> None:
super().validate(report)
operations = [
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.operation.name"
]
agent_names = {
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.agent.name"
}
assert operations.count("invoke_agent") >= 2, (
"Orchestration involves a triage agent handing off to a specialist; "
f"expected at least two invoke_agent spans, saw {operations}"
)
assert operations.count("execute_tool") >= 1, (
"Specialist agent calls the get_weather function tool; "
f"expected at least one execute_tool span, saw {operations}"
)
assert len(agent_names) >= 2, (
"Triage and specialist must each surface their own gen_ai.agent.name; "
f"saw {sorted(agent_names)}"
Expand Down
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


class EmbeddingScenario(Scenario):
expected_spans = ("embeddings",)
expected_spans = {"embeddings": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


class InferenceScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
17 changes: 1 addition & 16 deletions ...on/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_conversation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,14 @@
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.test.weaver_live_check import LiveCheckReport
from opentelemetry.test_util_genai.conformance import Scenario
from opentelemetry.test_util_genai.instrumentor import instrument

DEFAULT_MODEL = "gpt-4o-mini"


class ResponsesConversationScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 2}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down Expand Up @@ -54,17 +53,3 @@ def run(
input="What is my favorite color?",
previous_response_id=first.id,
)

def validate(self, report: LiveCheckReport) -> None:
super().validate(report)
operations = [
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.operation.name"
]
assert operations == ["chat", "chat"], (
"Responses conversation exercises two Responses API calls "
f"(initial request and follow-up); saw spans {operations}"
)
2 changes: 1 addition & 1 deletion ...entation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_stream.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@


class ResponsesStreamScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
17 changes: 1 addition & 16 deletions instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/tool_calling.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.test.weaver_live_check import LiveCheckReport
from opentelemetry.test_util_genai.conformance import Scenario
from opentelemetry.test_util_genai.instrumentor import instrument

Expand Down Expand Up @@ -60,7 +59,7 @@ def _execute_weather_tool(arguments: str) -> str:


class ToolCallingScenario(Scenario):
expected_spans = ("chat",)
expected_spans = {"chat": 2}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down Expand Up @@ -111,17 +110,3 @@ def run(
messages=messages,
model=DEFAULT_MODEL,
)

def validate(self, report: LiveCheckReport) -> None:
super().validate(report)
operations = [
attr["value"]
for entry in report["samples"]
if "span" in entry
for attr in entry["span"]["attributes"]
if attr["name"] == "gen_ai.operation.name"
]
assert operations == ["chat", "chat"], (
"Tool calling exercises two chat completions (initial request and "
f"follow-up with tool results); saw spans {operations}"
)
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-google-genai/tests/conformance/embedding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@


class EmbeddingScenario(Scenario):
expected_spans = ("embeddings",)
expected_spans = {"embeddings": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
2 changes: 1 addition & 1 deletion ...entation/opentelemetry-instrumentation-google-genai/tests/conformance/generate_content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@


class GenerateContentScenario(Scenario):
expected_spans = ("generate_content",)
expected_spans = {"generate_content": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-google-genai/tests/conformance/inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@


class InferenceScenario(Scenario):
expected_spans = ("interactions.create",)
expected_spans = {"interactions.create": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
2 changes: 1 addition & 1 deletion instrumentation/opentelemetry-instrumentation-google-genai/tests/conformance/tool_calling.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@


class ToolCallingScenario(Scenario):
expected_spans = ("generate_content",)
expected_spans = {"generate_content": 1}
expected_metrics = (
"gen_ai.client.operation.duration",
"gen_ai.client.token.usage",
Expand Down
Loading