diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/124.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/124.added new file mode 100644 index 00000000..97a29b7f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/124.added @@ -0,0 +1 @@ +Added retrieval span support. \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/pyproject.toml b/instrumentation/opentelemetry-instrumentation-genai-langchain/pyproject.toml index 5d300564..8bc70e53 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ "opentelemetry-instrumentation >= 0.62b0", - "opentelemetry-util-genai >= 1.0b0.dev", + "opentelemetry-util-genai >= 0.1b0", ] [project.optional-dependencies] diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 8779b3d4..6eca4ef6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -3,10 +3,12 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any, Optional, cast from uuid import UUID from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.documents import Document from langchain_core.messages import BaseMessage from langchain_core.outputs import LLMResult @@ -26,6 +28,7 @@ from opentelemetry.util.genai.invocation import ( AgentInvocation, InferenceInvocation, + RetrievalInvocation, WorkflowInvocation, ) from opentelemetry.util.genai.types import ( @@ -402,6 +405,78 @@ def on_llm_error( if not llm_invocation.span.is_recording(): self._invocation_manager.delete_invocation_state(run_id=run_id) + def on_retriever_start( + self, + serialized: dict[str, Any], + query: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + metadata: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> Any: + meta = metadata or {} + provider = meta.get("ls_vector_store_provider") or None + request_model = meta.get("ls_embedding_model") or None + retrieval = self._telemetry_handler.retrieval( + provider=provider, request_model=request_model + ) + retrieval.query_text = query + self._invocation_manager.add_invocation_state( + run_id, parent_run_id, retrieval + ) + + def on_retriever_end( + self, + documents: Sequence[Document], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + invocation = self._invocation_manager.get_invocation(run_id=run_id) + if invocation is None or not isinstance( + invocation, RetrievalInvocation + ): + self._invocation_manager.delete_invocation_state(run_id) + return + + invocation.documents = [ + { + "content": doc.page_content, + **({"id": doc.id} if doc.id is not None else {}), + **{ + k: v + for k, v in cast(dict[str, Any], doc.metadata).items() + if v is not None + }, + } + for doc in documents + ] + invocation.stop() + if not invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id) + + def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + invocation = self._invocation_manager.get_invocation(run_id=run_id) + if invocation is None or not isinstance( + invocation, RetrievalInvocation + ): + self._invocation_manager.delete_invocation_state(run_id) + return + + invocation.fail(error) + if not invocation.span.is_recording(): + self._invocation_manager.delete_invocation_state(run_id=run_id) + def _find_nearest_agent( self, run_id: Optional[UUID] ) -> Optional[AgentInvocation]: diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/retrieval.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/retrieval.py new file mode 100644 index 00000000..750df8f1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/retrieval.py @@ -0,0 +1,99 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: langchain retrieval via VectorStoreRetriever.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.callbacks import CallbackManagerForRetrieverRun +from langchain_core.documents import Document +from langchain_core.retrievers import BaseRetriever + +from opentelemetry.instrumentation.genai.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + + +class _FakeRetriever(BaseRetriever): + """In-memory retriever that returns fixed documents without network calls.""" + + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun + ) -> list[Document]: + return [ + Document( + page_content="Paris is the capital of France.", + id="doc-1", + metadata={"source": "wiki"}, + ), + Document( + page_content="The Eiffel Tower is located in Paris.", + id="doc-2", + metadata={"source": "wiki"}, + ), + ] + + def _get_ls_params(self, **kwargs: Any) -> Any: + params = super()._get_ls_params(**kwargs) + params["ls_vector_store_provider"] = "FakeVectorStore" + return params + + +class RetrievalScenario(Scenario): + expected_spans = ("retrieval",) + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = ( + # LangChain's Document type has no relevance score field; the + # instrumentation cannot populate gen_ai.retrieval.documents[].score. + ExpectedViolation( + advice_id="genai_content_schema", + message_substring="score", + ), + # _FakeRetriever is in-memory and has no backing server. + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + retriever = _FakeRetriever() + # No VCR cassette needed — _FakeRetriever makes no network calls. + retriever.invoke("What is the capital of France?") + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + operations = [ + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == "gen_ai.operation.name" + ] + assert "retrieval" in operations, ( + f"Expected a retrieval span; saw operations {operations}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index be5476e4..b416cf7e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -11,6 +11,7 @@ import uuid from unittest import mock +from langchain_core.documents import Document from langchain_core.messages import AIMessage, HumanMessage from opentelemetry.instrumentation.genai.langchain.callback_handler import ( @@ -24,6 +25,7 @@ ) from opentelemetry.util.genai.invocation import ( AgentInvocation, + RetrievalInvocation, WorkflowInvocation, ) from opentelemetry.util.genai.types import InputMessage, OutputMessage, Text @@ -55,6 +57,15 @@ def _side_effect(*args, **kwargs): return _side_effect +def _make_retrieval_inv_mock() -> mock.MagicMock: + retrieval_inv = mock.MagicMock(spec=RetrievalInvocation) + retrieval_inv.span = mock.MagicMock() + retrieval_inv.span.is_recording.return_value = False + retrieval_inv.query_text = None + retrieval_inv.documents = None + return retrieval_inv + + def _make_handler(): """Return a handler wired to a MagicMock TelemetryHandler.""" telemetry = mock.MagicMock() @@ -76,6 +87,14 @@ def _make_handler(): return handler, telemetry, workflow_inv, agent_inv +def _make_handler_with_retrieval(): + """Like _make_handler but also wires up a retrieval mock.""" + handler, telemetry, workflow_inv, agent_inv = _make_handler() + retrieval_inv = _make_retrieval_inv_mock() + telemetry.retrieval.return_value = retrieval_inv + return handler, telemetry, retrieval_inv + + def _run_id(): return uuid.uuid4() @@ -1015,3 +1034,207 @@ def test_agent_output_messages_only_last_ai_message(self): assigned = agent_inv.output_messages assert len(assigned) == 1 assert assigned[0].parts[0].content == "the answer is 7" + + +# --------------------------------------------------------------------------- +# on_retriever_start / on_retriever_end / on_retriever_error +# --------------------------------------------------------------------------- + + +class TestOnRetrieverStart: + def test_retrieval_span_created(self): + handler, telemetry, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="what is AI?", + run_id=run_id, + ) + + telemetry.retrieval.assert_called_once() + assert ( + handler._invocation_manager.get_invocation(run_id) is retrieval_inv + ) + + def test_query_text_set_on_invocation(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="semantic search query", + run_id=run_id, + ) + + assert retrieval_inv.query_text == "semantic search query" + + def test_provider_passed_from_metadata(self): + handler, telemetry, _ = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="q", + run_id=run_id, + metadata={"ls_vector_store_provider": "Chroma"}, + ) + + telemetry.retrieval.assert_called_once_with( + provider="Chroma", request_model=None + ) + + def test_provider_none_when_metadata_absent(self): + handler, telemetry, _ = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="q", + run_id=run_id, + ) + + telemetry.retrieval.assert_called_once_with( + provider=None, request_model=None + ) + + def test_request_model_passed_from_ls_embedding_model(self): + handler, telemetry, _ = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="q", + run_id=run_id, + metadata={ + "ls_vector_store_provider": "Chroma", + "ls_embedding_model": "text-embedding-3-small", + }, + ) + + telemetry.retrieval.assert_called_once_with( + provider="Chroma", request_model="text-embedding-3-small" + ) + + def test_request_model_none_when_ls_embedding_model_absent(self): + handler, telemetry, _ = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="q", + run_id=run_id, + metadata={"ls_vector_store_provider": "Chroma"}, + ) + + telemetry.retrieval.assert_called_once_with( + provider="Chroma", request_model=None + ) + + def test_registered_in_invocation_manager(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start( + serialized={}, + query="q", + run_id=run_id, + ) + + assert run_id in handler._invocation_manager._invocations + assert ( + handler._invocation_manager.get_invocation(run_id) is retrieval_inv + ) + + +class TestOnRetrieverEnd: + def test_invocation_stopped(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[], run_id=run_id) + + retrieval_inv.stop.assert_called_once() + + def test_documents_set_from_page_content(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + docs = [ + Document(page_content="doc one", metadata={"source": "s1"}), + Document(page_content="doc two", metadata={}), + ] + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=docs, run_id=run_id) + + assigned = retrieval_inv.documents + assert len(assigned) == 2 + assert assigned[0]["content"] == "doc one" + assert assigned[0]["source"] == "s1" + assert assigned[1]["content"] == "doc two" + + def test_document_id_included_when_present(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + doc = Document(page_content="text", id="doc-123", metadata={}) + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[doc], run_id=run_id) + + assert retrieval_inv.documents[0]["id"] == "doc-123" + + def test_document_id_absent_when_none(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + doc = Document(page_content="text", metadata={}) + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + handler.on_retriever_end(documents=[doc], run_id=run_id) + + assert "id" not in retrieval_inv.documents[0] + + def test_state_cleaned_up_after_end(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + retrieval_inv.span.is_recording.return_value = False + handler.on_retriever_end(documents=[], run_id=run_id) + + assert run_id not in handler._invocation_manager._invocations + + def test_unknown_run_id_does_not_raise(self): + handler, _, _ = _make_handler_with_retrieval() + handler.on_retriever_end(documents=[], run_id=_run_id()) + + +class TestOnRetrieverError: + def test_invocation_failed(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + err = RuntimeError("retrieval failed") + handler.on_retriever_error(error=err, run_id=run_id) + + retrieval_inv.fail.assert_called_once_with(err) + + def test_state_cleaned_up_after_error(self): + handler, _, retrieval_inv = _make_handler_with_retrieval() + run_id = _run_id() + + handler.on_retriever_start(serialized={}, query="q", run_id=run_id) + retrieval_inv.span.is_recording.return_value = False + handler.on_retriever_error(error=RuntimeError("boom"), run_id=run_id) + + assert run_id not in handler._invocation_manager._invocations + + def test_unknown_run_id_does_not_raise(self): + handler, _, _ = _make_handler_with_retrieval() + handler.on_retriever_error( + error=RuntimeError("boom"), run_id=_run_id() + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py index 3bdf216b..bf18ef2a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py @@ -22,6 +22,7 @@ from .conformance.agent import AgentScenario from .conformance.inference import InferenceScenario +from .conformance.retrieval import RetrievalScenario from .conformance.workflow import WorkflowScenario @@ -31,6 +32,7 @@ InferenceScenario(), AgentScenario(), WorkflowScenario(), + RetrievalScenario(), ], ids=lambda s: type(s).__name__, ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py new file mode 100644 index 00000000..5fccd499 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_retriever.py @@ -0,0 +1,365 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for retriever instrumentation. + +These tests exercise the full on_retriever_start / on_retriever_end / +on_retriever_error path end-to-end by driving LangChain's retriever +interface against an in-memory fake retriever and verifying the emitted +spans, attributes, and metrics against the semconv spec. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from langchain_core.callbacks import CallbackManagerForRetrieverRun +from langchain_core.documents import Document +from langchain_core.retrievers import BaseRetriever + +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import error_attributes + +# --------------------------------------------------------------------------- +# Fake retriever helpers +# --------------------------------------------------------------------------- + + +class _FakeRetriever(BaseRetriever): + """In-memory retriever — no network calls, no embeddings.""" + + documents: list[Document] = [] + + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun + ) -> list[Document]: + return self.documents + + def _get_ls_params(self, **kwargs: Any) -> Any: + params = super()._get_ls_params(**kwargs) + params["ls_vector_store_provider"] = "FakeVectorStore" + return params + + +class _ErrorRetriever(BaseRetriever): + """Retriever that always raises.""" + + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun + ) -> list[Document]: + raise RuntimeError("retrieval failed") + + def _get_ls_params(self, **kwargs: Any) -> Any: + params = super()._get_ls_params(**kwargs) + params["ls_vector_store_provider"] = "FakeVectorStore" + return params + + +# --------------------------------------------------------------------------- +# Happy-path span attributes +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "capture_content", + ["SPAN_ONLY", "NO_CONTENT", "SPAN_AND_EVENT", "EVENT_ONLY"], +) +def test_retrieval_span_attributes( + span_exporter, + metric_reader, + start_instrumentation, + monkeypatch, + capture_content, +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content + ) + + docs = [ + Document(page_content="Paris is the capital of France.", id="doc-1"), + Document( + page_content="Berlin is the capital of Germany.", + metadata={"source": "wiki"}, + ), + ] + retriever = _FakeRetriever(documents=docs) + + result = retriever.invoke("What is the capital of France?") + assert len(result) == 2 + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.name == "retrieval" + attrs = span.attributes + assert attrs[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "retrieval" + assert attrs[gen_ai_attributes.GEN_AI_PROVIDER_NAME] == "FakeVectorStore" + + should_capture = capture_content in ("SPAN_ONLY", "SPAN_AND_EVENT") + if should_capture: + assert ( + attrs[gen_ai_attributes.GEN_AI_RETRIEVAL_QUERY_TEXT] + == "What is the capital of France?" + ) + docs_attr = attrs[gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS] + assert docs_attr is not None + assert "Paris is the capital of France." in docs_attr + assert "doc-1" in docs_attr + assert "Berlin is the capital of Germany." in docs_attr + assert "wiki" in docs_attr + else: + assert gen_ai_attributes.GEN_AI_RETRIEVAL_QUERY_TEXT not in attrs + assert gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS not in attrs + + +def test_retrieval_span_name_without_data_source_id( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + retriever = _FakeRetriever(documents=[]) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert spans[0].name == "retrieval" + + +def test_retrieval_span_no_model_when_ls_embedding_model_absent( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + retriever = _FakeRetriever(documents=[]) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert gen_ai_attributes.GEN_AI_REQUEST_MODEL not in spans[0].attributes + + +def test_retrieval_span_model_set_when_ls_embedding_model_present( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + + class _RetrieverWithModel(_FakeRetriever): + def _get_ls_params(self, **kwargs: Any) -> Any: + params = super()._get_ls_params(**kwargs) + params["ls_embedding_model"] = "text-embedding-3-small" + return params + + retriever = _RetrieverWithModel(documents=[]) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert ( + spans[0].attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + == "text-embedding-3-small" + ) + + +# --------------------------------------------------------------------------- +# Error path +# --------------------------------------------------------------------------- + + +def test_retrieval_error_span( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + retriever = _ErrorRetriever() + + with pytest.raises(RuntimeError, match="retrieval failed"): + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.name == "retrieval" + attrs = span.attributes + assert attrs[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "retrieval" + assert attrs[gen_ai_attributes.GEN_AI_PROVIDER_NAME] == "FakeVectorStore" + assert attrs[error_attributes.ERROR_TYPE] == "RuntimeError" + + +# --------------------------------------------------------------------------- +# Duration metric +# --------------------------------------------------------------------------- + + +def test_retrieval_duration_metric_emitted( + span_exporter, metric_reader, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + retriever = _FakeRetriever(documents=[Document(page_content="content")]) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + metrics_data = metric_reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + assert len(resource_metrics) == 1 + + all_metrics = resource_metrics[0].scope_metrics[0].metrics + duration_metrics = [ + m + for m in all_metrics + if m.name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert len(duration_metrics) == 1 + + dp = duration_metrics[0].data.data_points + assert len(dp) == 1 + assert dp[0].sum > 0 + + metric_attrs = dp[0].attributes + assert metric_attrs[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "retrieval" + assert ( + metric_attrs[gen_ai_attributes.GEN_AI_PROVIDER_NAME] + == "FakeVectorStore" + ) + + # Exemplar links back to the span + assert len(dp[0].exemplars) == 1 + assert dp[0].exemplars[0].span_id == span.get_span_context().span_id + assert dp[0].exemplars[0].trace_id == span.get_span_context().trace_id + + +def test_retrieval_error_duration_metric_emitted( + span_exporter, metric_reader, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + retriever = _ErrorRetriever() + + with pytest.raises(RuntimeError): + retriever.invoke("q") + + metrics_data = metric_reader.get_metrics_data() + resource_metrics = metrics_data.resource_metrics + assert len(resource_metrics) == 1 + + all_metrics = resource_metrics[0].scope_metrics[0].metrics + duration_metrics = [ + m + for m in all_metrics + if m.name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert len(duration_metrics) == 1 + + dp = duration_metrics[0].data.data_points + assert len(dp) == 1 + assert dp[0].sum > 0 + + metric_attrs = dp[0].attributes + assert metric_attrs[error_attributes.ERROR_TYPE] == "RuntimeError" + assert metric_attrs[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "retrieval" + + +# --------------------------------------------------------------------------- +# Document mapping +# --------------------------------------------------------------------------- + + +def test_document_id_in_span_content( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + docs = [Document(page_content="text", id="abc-123", metadata={})] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + assert "abc-123" in docs_attr + + +def test_document_without_id_in_span_content( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + docs = [Document(page_content="no id here", metadata={})] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + assert "no id here" in docs_attr + assert '"id"' not in docs_attr + + +def test_document_metadata_in_span_content( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + docs = [ + Document( + page_content="text", + metadata={"source": "wiki", "score": 0.9}, + ) + ] + retriever = _FakeRetriever(documents=docs) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + docs_attr = spans[0].attributes[ + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ] + assert "wiki" in docs_attr + assert "0.9" in docs_attr + + +def test_empty_documents_in_span_content( + span_exporter, start_instrumentation, monkeypatch +): + monkeypatch.setenv( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" + ) + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + retriever = _FakeRetriever(documents=[]) + retriever.invoke("q") + + spans = span_exporter.get_finished_spans() + # documents attribute is set but represents an empty list + docs_attr = spans[0].attributes.get( + gen_ai_attributes.GEN_AI_RETRIEVAL_DOCUMENTS + ) + assert docs_attr == "[]"