diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py index 8f642567ca..d694857da4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Optional, cast from uuid import UUID from langchain_core.callbacks import BaseCallbackHandler @@ -29,6 +29,7 @@ Error, InputMessage, LLMInvocation, + MessagePart, OutputMessage, Text, ) @@ -133,7 +134,11 @@ def on_chat_model_start( Text(content=text_value, type="text") ) - input_messages.append(InputMessage(parts=parts, role=role)) + input_messages.append( + InputMessage( + parts=cast(list[MessagePart], parts), role=role + ) + ) llm_invocation = LLMInvocation( request_model=request_model, @@ -206,7 +211,7 @@ def on_llm_end( role = chat_generation.message.type output_message = OutputMessage( role=role, - parts=parts, + parts=cast(list[MessagePart], parts), finish_reason=finish_reason, ) output_messages.append(output_message) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler_memory.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler_memory.py new file mode 100644 index 0000000000..484c0819e2 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_callback_handler_memory.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from unittest import mock +from uuid import uuid4 + +from opentelemetry.instrumentation.langchain.callback_handler import ( + GEN_AI_MEMORY_QUERY, + GEN_AI_MEMORY_SEARCH_RESULT_COUNT, + GEN_AI_MEMORY_STORE_ID, + GEN_AI_MEMORY_STORE_NAME, + RETRIEVAL_OPERATION, + SEARCH_MEMORY_OPERATION, + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.util.genai.types import ContentCapturingMode + + +def _build_handler(): + telemetry_handler = mock.Mock() + + def _start(invocation): + span = mock.Mock() + span.is_recording.return_value = True + invocation.span = span + return invocation + + telemetry_handler.start_llm.side_effect = _start + telemetry_handler.stop_llm.side_effect = lambda invocation: invocation + telemetry_handler.fail_llm.side_effect = ( + lambda invocation, error: invocation + ) + return ( + OpenTelemetryLangChainCallbackHandler(telemetry_handler), + telemetry_handler, + ) + + +def test_retriever_defaults_to_retrieval_without_memory_metadata(monkeypatch): + """Retrievers without memory metadata should emit 'retrieval' operation.""" + handler, telemetry_handler = _build_handler() + monkeypatch.setattr( + "opentelemetry.instrumentation.langchain.callback_handler.is_experimental_mode", + lambda: False, + ) + + run_id = uuid4() + handler.on_retriever_start( + serialized={"name": "PineconeRetriever"}, + query="what is RAG?", + run_id=run_id, + metadata={"ls_provider": "pinecone"}, + ) + + invocation = handler._invocation_manager.get_invocation(run_id) + assert invocation is not None + assert invocation.operation_name == RETRIEVAL_OPERATION + telemetry_handler.start_llm.assert_called_once() + + +def test_retriever_uses_search_memory_with_memory_metadata(monkeypatch): + """Retrievers with memory_store_name in metadata should emit 'search_memory'.""" + handler, telemetry_handler = _build_handler() + monkeypatch.setattr( + "opentelemetry.instrumentation.langchain.callback_handler.is_experimental_mode", + lambda: True, + ) + monkeypatch.setattr( + "opentelemetry.instrumentation.langchain.callback_handler.get_content_capturing_mode", + lambda: ContentCapturingMode.SPAN_ONLY, + ) + + run_id = uuid4() + handler.on_retriever_start( + serialized={ + "name": "SessionMemoryRetriever", + "id": ["langchain", "retriever", "session"], + }, + query="user preferences", + run_id=run_id, + metadata={ + "ls_provider": "openai", + "memory_store_name": "SessionMemoryRetriever", + "memory_namespace": "user-123", + }, + ) + + invocation = handler._invocation_manager.get_invocation(run_id) + assert invocation is not None + assert invocation.operation_name == SEARCH_MEMORY_OPERATION + assert ( + invocation.attributes[GEN_AI_MEMORY_STORE_NAME] + == "SessionMemoryRetriever" + ) + assert ( + invocation.attributes[GEN_AI_MEMORY_STORE_ID] + == "langchain.retriever.session" + ) + assert invocation.attributes[GEN_AI_MEMORY_QUERY] == "user preferences" + telemetry_handler.start_llm.assert_called_once() + + +def test_on_retriever_end_sets_search_result_count(monkeypatch): + handler, telemetry_handler = _build_handler() + monkeypatch.setattr( + "opentelemetry.instrumentation.langchain.callback_handler.is_experimental_mode", + lambda: False, + ) + + run_id = uuid4() + handler.on_retriever_start( + serialized={"name": "MemoryRetriever"}, + query="q", + run_id=run_id, + metadata={"ls_provider": "openai"}, + ) + handler.on_retriever_end(documents=[object(), object()], run_id=run_id) + + telemetry_handler.stop_llm.assert_called_once() + stop_invocation = telemetry_handler.stop_llm.call_args.kwargs["invocation"] + assert stop_invocation.attributes[GEN_AI_MEMORY_SEARCH_RESULT_COUNT] == 2 + + +def test_on_retriever_error_fails_invocation(monkeypatch): + handler, telemetry_handler = _build_handler() + monkeypatch.setattr( + "opentelemetry.instrumentation.langchain.callback_handler.is_experimental_mode", + lambda: False, + ) + + run_id = uuid4() + handler.on_retriever_start( + serialized={"name": "VectorRetriever"}, + query="q", + run_id=run_id, + metadata={"ls_provider": "openai"}, + ) + handler.on_retriever_error(RuntimeError("retrieval failed"), run_id=run_id) + + telemetry_handler.fail_llm.assert_called_once() + fail_invocation = telemetry_handler.fail_llm.call_args.kwargs["invocation"] + assert fail_invocation.operation_name == RETRIEVAL_OPERATION diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-mem0/CHANGELOG.md new file mode 100644 index 0000000000..b22e5f411d --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +- Initial release: Mem0 memory operation instrumentation aligned with GenAI + memory semantic conventions. + ([#3250](https://github.com/open-telemetry/semantic-conventions/pull/3250)) diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/LICENSE b/instrumentation-genai/opentelemetry-instrumentation-mem0/LICENSE new file mode 100644 index 0000000000..eba7b73574 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/LICENSE @@ -0,0 +1,7 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + See https://www.apache.org/licenses/LICENSE-2.0 for the full license text. diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/README.rst b/instrumentation-genai/opentelemetry-instrumentation-mem0/README.rst new file mode 100644 index 0000000000..6a605b14e7 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/README.rst @@ -0,0 +1,39 @@ +OpenTelemetry Mem0 Instrumentation +=================================== + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-mem0.svg + :target: https://pypi.org/project/opentelemetry-instrumentation-mem0/ + +This library allows tracing Mem0 memory operations (add, search, update, +delete) using OpenTelemetry, emitting spans with GenAI memory semantic +convention attributes. + +Installation +------------ + +:: + + pip install opentelemetry-instrumentation-mem0 + +Usage +----- + +.. code-block:: python + + from opentelemetry.instrumentation.mem0 import Mem0Instrumentor + from mem0 import Memory + + Mem0Instrumentor().instrument() + + m = Memory() + m.add("I prefer dark mode", user_id="alice") + results = m.search("preferences", user_id="alice") + +References +---------- + +* `OpenTelemetry Project `_ +* `Mem0 `_ +* `GenAI Memory Semantic Conventions `_ diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-mem0/pyproject.toml new file mode 100644 index 0000000000..e01ddef095 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-instrumentation-mem0" +dynamic = ["version"] +description = "OpenTelemetry Mem0 instrumentation" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "opentelemetry-api ~= 1.37", + "opentelemetry-instrumentation ~= 0.58b0", + "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-util-genai ~= 0.58b0", +] + +[project.optional-dependencies] +instruments = [ + "mem0ai >= 0.1.0", +] + +[project.entry-points.opentelemetry_instrumentor] +mem0 = "opentelemetry.instrumentation.mem0:Mem0Instrumentor" + +[project.urls] +Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation-genai/opentelemetry-instrumentation-mem0" +Repository = "https://github.com/open-telemetry/opentelemetry-python-contrib" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/mem0/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/examples", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/__init__.py new file mode 100644 index 0000000000..145fac804a --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/__init__.py @@ -0,0 +1,124 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenTelemetry Mem0 Instrumentation +=================================== + +Instrumentation for the Mem0 Python SDK memory operations, aligned with +the `GenAI memory semantic conventions +`_. + +Usage +----- + +.. code-block:: python + + from opentelemetry.instrumentation.mem0 import Mem0Instrumentor + from mem0 import Memory + + Mem0Instrumentor().instrument() + + m = Memory() + m.add("I prefer dark mode", user_id="alice") + results = m.search("preferences", user_id="alice") + +Configuration +------------- + +Memory content capture can be enabled by setting the environment variable: +``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`` + +API +--- +""" + +from typing import Any, Collection + +from wrapt import wrap_function_wrapper + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.mem0.package import _instruments +from opentelemetry.instrumentation.mem0.patch import ( + wrap_memory_add, + wrap_memory_delete, + wrap_memory_delete_all, + wrap_memory_get_all, + wrap_memory_search, + wrap_memory_update, +) +from opentelemetry.instrumentation.utils import unwrap + + +class Mem0Instrumentor(BaseInstrumentor): + """An instrumentor for the Mem0 Python SDK. + + Traces Mem0 memory operations (add, search, update, delete) and emits + spans with GenAI memory semantic convention attributes plus a + ``gen_ai.client.operation.duration`` histogram metric. + """ + + def __init__(self) -> None: + super().__init__() + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any) -> None: + tracer_provider = kwargs.get("tracer_provider") + meter_provider = kwargs.get("meter_provider") + + wrap_function_wrapper( + module="mem0.memory.main", + name="Memory.add", + wrapper=wrap_memory_add(tracer_provider, meter_provider), + ) + wrap_function_wrapper( + module="mem0.memory.main", + name="Memory.search", + wrapper=wrap_memory_search(tracer_provider, meter_provider), + ) + wrap_function_wrapper( + module="mem0.memory.main", + name="Memory.update", + wrapper=wrap_memory_update(tracer_provider, meter_provider), + ) + wrap_function_wrapper( + module="mem0.memory.main", + name="Memory.delete", + wrapper=wrap_memory_delete(tracer_provider, meter_provider), + ) + wrap_function_wrapper( + module="mem0.memory.main", + name="Memory.delete_all", + wrapper=wrap_memory_delete_all(tracer_provider, meter_provider), + ) + wrap_function_wrapper( + module="mem0.memory.main", + name="Memory.get_all", + wrapper=wrap_memory_get_all(tracer_provider, meter_provider), + ) + + def _uninstrument(self, **kwargs: Any) -> None: + import mem0.memory.main # noqa: PLC0415 + + for method in ( + "add", + "search", + "update", + "delete", + "delete_all", + "get_all", + ): + unwrap(mem0.memory.main.Memory, method) diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/package.py b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/package.py new file mode 100644 index 0000000000..2d4c16baf3 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/package.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_instruments = ("mem0ai >= 0.1.0",) diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/patch.py b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/patch.py new file mode 100644 index 0000000000..26a6d3cac6 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/patch.py @@ -0,0 +1,409 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Wrapping functions for Mem0 Memory class methods. + +Each wrapper emits a CLIENT span with GenAI memory semantic convention +attributes, an ``error.type`` attribute on failure, and records a +``gen_ai.client.operation.duration`` histogram metric. +""" + +from __future__ import annotations + +import os +import timeit +from typing import Any, Callable, Optional + +from opentelemetry import trace +from opentelemetry.metrics import MeterProvider, get_meter +from opentelemetry.semconv._incubating.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.trace import SpanKind, StatusCode +from opentelemetry.util.genai.instruments import create_duration_histogram + +_INSTRUMENTATION_NAME = "opentelemetry.instrumentation.mem0" + +# --------------------------------------------------------------------------- +# Attribute constants — resolved at import time so they stay in sync with +# whichever semconv version is installed. +# --------------------------------------------------------------------------- + + +def _attr(name: str, fallback: str) -> str: + return getattr(GenAIAttributes, name, fallback) + + +GEN_AI_OPERATION_NAME = _attr("GEN_AI_OPERATION_NAME", "gen_ai.operation.name") +GEN_AI_SYSTEM = _attr("GEN_AI_SYSTEM", "gen_ai.system") +GEN_AI_PROVIDER_NAME = _attr("GEN_AI_PROVIDER_NAME", "gen_ai.provider.name") +GEN_AI_MEMORY_STORE_ID = _attr( + "GEN_AI_MEMORY_STORE_ID", "gen_ai.memory.store.id" +) +GEN_AI_MEMORY_RECORD_ID = _attr( + "GEN_AI_MEMORY_RECORD_ID", "gen_ai.memory.record.id" +) +GEN_AI_MEMORY_RECORD_CONTENT = _attr( + "GEN_AI_MEMORY_RECORD_CONTENT", "gen_ai.memory.record.content" +) +GEN_AI_MEMORY_QUERY_TEXT = _attr( + "GEN_AI_MEMORY_QUERY_TEXT", "gen_ai.memory.query.text" +) +GEN_AI_MEMORY_SEARCH_RESULT_COUNT = _attr( + "GEN_AI_MEMORY_SEARCH_RESULT_COUNT", "gen_ai.memory.search.result.count" +) +ERROR_TYPE = getattr(ErrorAttributes, "ERROR_TYPE", "error.type") + +_PROVIDER = "mem0" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _capture_content() -> bool: + return os.environ.get( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "" + ).lower() in ("true", "1") + + +def _set_common_attributes( + span: trace.Span, + operation: str, + kwargs: dict[str, Any], +) -> None: + """Set attributes shared by all memory operations.""" + span.set_attribute(GEN_AI_OPERATION_NAME, operation) + span.set_attribute(GEN_AI_SYSTEM, _PROVIDER) + span.set_attribute(GEN_AI_PROVIDER_NAME, _PROVIDER) + + +def _set_error(span: trace.Span, exc: BaseException) -> str: + """Record error details on the span and return the error type string.""" + error_type = type(exc).__qualname__ + span.set_status(StatusCode.ERROR, str(exc)) + span.set_attribute(ERROR_TYPE, error_type) + span.record_exception(exc) + return error_type + + +def _record_duration( + duration_histogram, + duration_s: float, + operation: str, + error_type: Optional[str] = None, +) -> None: + """Record the operation duration metric.""" + if duration_histogram is None: + return + attrs: dict[str, Any] = { + GEN_AI_OPERATION_NAME: operation, + GEN_AI_SYSTEM: _PROVIDER, + } + if error_type: + attrs[ERROR_TYPE] = error_type + duration_histogram.record(max(duration_s, 0.0), attributes=attrs) + + +def _result_count(result: Any) -> Optional[int]: + """Extract result count from a Mem0 response (dict or list).""" + if isinstance(result, dict) and "results" in result: + return len(result["results"]) + if isinstance(result, list): + return len(result) + return None + + +def _first_memory_id(result: Any) -> Optional[str]: + """Extract the first memory id from an add/update result.""" + if isinstance(result, dict) and result.get("results"): + for item in result["results"]: + if isinstance(item, dict) and item.get("id"): + return str(item["id"]) + return None + + +# --------------------------------------------------------------------------- +# Wrapper factories +# --------------------------------------------------------------------------- + + +def wrap_memory_add( + tracer_provider: Optional[trace.TracerProvider] = None, + meter_provider: Optional[MeterProvider] = None, +) -> Callable: + tracer = trace.get_tracer( + _INSTRUMENTATION_NAME, tracer_provider=tracer_provider + ) + meter = get_meter(_INSTRUMENTATION_NAME, meter_provider=meter_provider) + duration_histogram = create_duration_histogram(meter) + + def wrapper( + wrapped: Callable, instance: Any, args: Any, kwargs: Any + ) -> Any: + span_name = f"update_memory {_PROVIDER}" + error_type = None + start = timeit.default_timer() + with tracer.start_as_current_span( + span_name, kind=SpanKind.CLIENT + ) as span: + _set_common_attributes(span, "update_memory", kwargs) + + if _capture_content() and args: + messages = args[0] if args else kwargs.get("messages") + if messages and isinstance(messages, str): + span.set_attribute(GEN_AI_MEMORY_RECORD_CONTENT, messages) + + try: + result = wrapped(*args, **kwargs) + except Exception as exc: + error_type = _set_error(span, exc) + raise + finally: + _record_duration( + duration_histogram, + timeit.default_timer() - start, + "update_memory", + error_type, + ) + + mem_id = _first_memory_id(result) + if mem_id: + span.set_attribute(GEN_AI_MEMORY_RECORD_ID, mem_id) + + return result + + return wrapper + + +def wrap_memory_search( + tracer_provider: Optional[trace.TracerProvider] = None, + meter_provider: Optional[MeterProvider] = None, +) -> Callable: + tracer = trace.get_tracer( + _INSTRUMENTATION_NAME, tracer_provider=tracer_provider + ) + meter = get_meter(_INSTRUMENTATION_NAME, meter_provider=meter_provider) + duration_histogram = create_duration_histogram(meter) + + def wrapper( + wrapped: Callable, instance: Any, args: Any, kwargs: Any + ) -> Any: + span_name = f"search_memory {_PROVIDER}" + error_type = None + start = timeit.default_timer() + with tracer.start_as_current_span( + span_name, kind=SpanKind.CLIENT + ) as span: + _set_common_attributes(span, "search_memory", kwargs) + + query = args[0] if args else kwargs.get("query") + if _capture_content() and query and isinstance(query, str): + span.set_attribute(GEN_AI_MEMORY_QUERY_TEXT, query) + + try: + result = wrapped(*args, **kwargs) + except Exception as exc: + error_type = _set_error(span, exc) + raise + finally: + _record_duration( + duration_histogram, + timeit.default_timer() - start, + "search_memory", + error_type, + ) + + count = _result_count(result) + if count is not None: + span.set_attribute(GEN_AI_MEMORY_SEARCH_RESULT_COUNT, count) + + return result + + return wrapper + + +def wrap_memory_update( + tracer_provider: Optional[trace.TracerProvider] = None, + meter_provider: Optional[MeterProvider] = None, +) -> Callable: + tracer = trace.get_tracer( + _INSTRUMENTATION_NAME, tracer_provider=tracer_provider + ) + meter = get_meter(_INSTRUMENTATION_NAME, meter_provider=meter_provider) + duration_histogram = create_duration_histogram(meter) + + def wrapper( + wrapped: Callable, instance: Any, args: Any, kwargs: Any + ) -> Any: + memory_id = args[0] if args else kwargs.get("memory_id") + span_name = f"update_memory {_PROVIDER}" + error_type = None + start = timeit.default_timer() + with tracer.start_as_current_span( + span_name, kind=SpanKind.CLIENT + ) as span: + _set_common_attributes(span, "update_memory", kwargs) + if memory_id: + span.set_attribute(GEN_AI_MEMORY_RECORD_ID, str(memory_id)) + + if _capture_content(): + data = kwargs.get("data") + if data and isinstance(data, str): + span.set_attribute(GEN_AI_MEMORY_RECORD_CONTENT, data) + + try: + result = wrapped(*args, **kwargs) + except Exception as exc: + error_type = _set_error(span, exc) + raise + finally: + _record_duration( + duration_histogram, + timeit.default_timer() - start, + "update_memory", + error_type, + ) + + return result + + return wrapper + + +def wrap_memory_delete( + tracer_provider: Optional[trace.TracerProvider] = None, + meter_provider: Optional[MeterProvider] = None, +) -> Callable: + tracer = trace.get_tracer( + _INSTRUMENTATION_NAME, tracer_provider=tracer_provider + ) + meter = get_meter(_INSTRUMENTATION_NAME, meter_provider=meter_provider) + duration_histogram = create_duration_histogram(meter) + + def wrapper( + wrapped: Callable, instance: Any, args: Any, kwargs: Any + ) -> Any: + memory_id = args[0] if args else kwargs.get("memory_id") + span_name = f"delete_memory {_PROVIDER}" + error_type = None + start = timeit.default_timer() + with tracer.start_as_current_span( + span_name, kind=SpanKind.CLIENT + ) as span: + _set_common_attributes(span, "delete_memory", kwargs) + if memory_id: + span.set_attribute(GEN_AI_MEMORY_RECORD_ID, str(memory_id)) + + try: + result = wrapped(*args, **kwargs) + except Exception as exc: + error_type = _set_error(span, exc) + raise + finally: + _record_duration( + duration_histogram, + timeit.default_timer() - start, + "delete_memory", + error_type, + ) + + return result + + return wrapper + + +def wrap_memory_delete_all( + tracer_provider: Optional[trace.TracerProvider] = None, + meter_provider: Optional[MeterProvider] = None, +) -> Callable: + tracer = trace.get_tracer( + _INSTRUMENTATION_NAME, tracer_provider=tracer_provider + ) + meter = get_meter(_INSTRUMENTATION_NAME, meter_provider=meter_provider) + duration_histogram = create_duration_histogram(meter) + + def wrapper( + wrapped: Callable, instance: Any, args: Any, kwargs: Any + ) -> Any: + span_name = f"delete_memory {_PROVIDER}" + error_type = None + start = timeit.default_timer() + with tracer.start_as_current_span( + span_name, kind=SpanKind.CLIENT + ) as span: + _set_common_attributes(span, "delete_memory", kwargs) + + try: + result = wrapped(*args, **kwargs) + except Exception as exc: + error_type = _set_error(span, exc) + raise + finally: + _record_duration( + duration_histogram, + timeit.default_timer() - start, + "delete_memory", + error_type, + ) + + return result + + return wrapper + + +def wrap_memory_get_all( + tracer_provider: Optional[trace.TracerProvider] = None, + meter_provider: Optional[MeterProvider] = None, +) -> Callable: + tracer = trace.get_tracer( + _INSTRUMENTATION_NAME, tracer_provider=tracer_provider + ) + meter = get_meter(_INSTRUMENTATION_NAME, meter_provider=meter_provider) + duration_histogram = create_duration_histogram(meter) + + def wrapper( + wrapped: Callable, instance: Any, args: Any, kwargs: Any + ) -> Any: + span_name = f"search_memory {_PROVIDER}" + error_type = None + start = timeit.default_timer() + with tracer.start_as_current_span( + span_name, kind=SpanKind.CLIENT + ) as span: + _set_common_attributes(span, "search_memory", kwargs) + + try: + result = wrapped(*args, **kwargs) + except Exception as exc: + error_type = _set_error(span, exc) + raise + finally: + _record_duration( + duration_histogram, + timeit.default_timer() - start, + "search_memory", + error_type, + ) + + count = _result_count(result) + if count is not None: + span.set_attribute(GEN_AI_MEMORY_SEARCH_RESULT_COUNT, count) + + return result + + return wrapper diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/version.py b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/version.py new file mode 100644 index 0000000000..e7bf4a48eb --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/src/opentelemetry/instrumentation/mem0/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1b0.dev" diff --git a/instrumentation-genai/opentelemetry-instrumentation-mem0/tests/test_mem0_instrumentation.py b/instrumentation-genai/opentelemetry-instrumentation-mem0/tests/test_mem0_instrumentation.py new file mode 100644 index 0000000000..b54f5036c9 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-mem0/tests/test_mem0_instrumentation.py @@ -0,0 +1,302 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for Mem0 memory operation instrumentation.""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from opentelemetry.instrumentation.mem0.patch import ( + ERROR_TYPE, + GEN_AI_MEMORY_QUERY_TEXT, + GEN_AI_MEMORY_RECORD_CONTENT, + GEN_AI_MEMORY_RECORD_ID, + GEN_AI_MEMORY_SEARCH_RESULT_COUNT, + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_SYSTEM, + wrap_memory_add, + wrap_memory_delete, + wrap_memory_delete_all, + wrap_memory_get_all, + wrap_memory_search, + wrap_memory_update, +) +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) +from opentelemetry.trace import SpanKind, StatusCode + + +class _InMemoryExporter(SpanExporter): + def __init__(self): + self.spans = [] + + def export(self, spans): + self.spans.extend(spans) + return SpanExportResult.SUCCESS + + def shutdown(self): + pass + + def get_finished_spans(self): + return list(self.spans) + + +@pytest.fixture() +def tracer_provider(): + provider = TracerProvider() + return provider + + +@pytest.fixture() +def exporter(tracer_provider): + exp = _InMemoryExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exp)) + return exp + + +@pytest.fixture() +def metric_reader(): + return InMemoryMetricReader() + + +@pytest.fixture() +def meter_provider(metric_reader): + return MeterProvider(metric_readers=[metric_reader]) + + +def _get_attrs(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + return span, {k: v for k, v in span.attributes.items()} + + +def _get_duration_metric(metric_reader): + """Return the recorded duration data points.""" + metrics = metric_reader.get_metrics_data() + for resource_metric in metrics.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + if metric.name == "gen_ai.client.operation.duration": + return metric.data.data_points + return [] + + +class TestMemoryAdd: + def test_basic_add( + self, tracer_provider, exporter, meter_provider, metric_reader + ): + wrapped = mock.Mock(return_value={"results": [{"id": "mem-1"}]}) + wrapper = wrap_memory_add(tracer_provider, meter_provider) + result = wrapper( + wrapped, None, ("I like dark mode",), {"user_id": "alice"} + ) + + assert result == {"results": [{"id": "mem-1"}]} + wrapped.assert_called_once() + + span, attrs = _get_attrs(exporter) + assert span.name == "update_memory mem0" + assert span.kind == SpanKind.CLIENT + assert attrs[GEN_AI_OPERATION_NAME] == "update_memory" + assert attrs[GEN_AI_SYSTEM] == "mem0" + assert attrs[GEN_AI_PROVIDER_NAME] == "mem0" + assert attrs[GEN_AI_MEMORY_RECORD_ID] == "mem-1" + + # Duration metric recorded + points = _get_duration_metric(metric_reader) + assert len(points) >= 1 + + def test_add_captures_content_when_enabled( + self, tracer_provider, exporter, meter_provider, monkeypatch + ): + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true" + ) + wrapped = mock.Mock(return_value={"results": []}) + wrapper = wrap_memory_add(tracer_provider, meter_provider) + wrapper(wrapped, None, ("I like dark mode",), {"user_id": "bob"}) + + span, attrs = _get_attrs(exporter) + assert attrs[GEN_AI_MEMORY_RECORD_CONTENT] == "I like dark mode" + + def test_add_does_not_capture_content_by_default( + self, tracer_provider, exporter, meter_provider, monkeypatch + ): + monkeypatch.delenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False + ) + wrapped = mock.Mock(return_value={"results": []}) + wrapper = wrap_memory_add(tracer_provider, meter_provider) + wrapper(wrapped, None, ("secret data",), {"user_id": "bob"}) + + span, attrs = _get_attrs(exporter) + assert GEN_AI_MEMORY_RECORD_CONTENT not in attrs + + def test_add_error_records_exception_and_type( + self, tracer_provider, exporter, meter_provider, metric_reader + ): + wrapped = mock.Mock(side_effect=RuntimeError("db down")) + wrapper = wrap_memory_add(tracer_provider, meter_provider) + + with pytest.raises(RuntimeError, match="db down"): + wrapper(wrapped, None, ("data",), {"user_id": "alice"}) + + span, attrs = _get_attrs(exporter) + assert span.status.status_code == StatusCode.ERROR + assert attrs[ERROR_TYPE] == "RuntimeError" + + # Duration metric still recorded on error + points = _get_duration_metric(metric_reader) + assert len(points) >= 1 + + +class TestMemorySearch: + def test_basic_search( + self, tracer_provider, exporter, meter_provider, metric_reader + ): + wrapped = mock.Mock( + return_value={"results": [{"id": "r1"}, {"id": "r2"}]} + ) + wrapper = wrap_memory_search(tracer_provider, meter_provider) + result = wrapper(wrapped, None, ("preferences",), {"user_id": "alice"}) + + assert result == {"results": [{"id": "r1"}, {"id": "r2"}]} + + span, attrs = _get_attrs(exporter) + assert span.name == "search_memory mem0" + assert attrs[GEN_AI_OPERATION_NAME] == "search_memory" + assert attrs[GEN_AI_PROVIDER_NAME] == "mem0" + assert attrs[GEN_AI_MEMORY_SEARCH_RESULT_COUNT] == 2 + + points = _get_duration_metric(metric_reader) + assert len(points) >= 1 + + def test_search_captures_query_when_enabled( + self, tracer_provider, exporter, meter_provider, monkeypatch + ): + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true" + ) + wrapped = mock.Mock(return_value=[]) + wrapper = wrap_memory_search(tracer_provider, meter_provider) + wrapper(wrapped, None, ("my query",), {"agent_id": "bot-1"}) + + span, attrs = _get_attrs(exporter) + assert attrs[GEN_AI_MEMORY_QUERY_TEXT] == "my query" + + def test_search_list_result( + self, tracer_provider, exporter, meter_provider + ): + wrapped = mock.Mock( + return_value=[{"id": "a"}, {"id": "b"}, {"id": "c"}] + ) + wrapper = wrap_memory_search(tracer_provider, meter_provider) + wrapper(wrapped, None, ("q",), {}) + + span, attrs = _get_attrs(exporter) + assert attrs[GEN_AI_MEMORY_SEARCH_RESULT_COUNT] == 3 + + +class TestMemoryUpdate: + def test_basic_update( + self, tracer_provider, exporter, meter_provider, metric_reader + ): + wrapped = mock.Mock(return_value={"status": "ok"}) + wrapper = wrap_memory_update(tracer_provider, meter_provider) + result = wrapper(wrapped, None, ("mem-42",), {"data": "new content"}) + + assert result == {"status": "ok"} + + span, attrs = _get_attrs(exporter) + assert span.name == "update_memory mem0" + assert attrs[GEN_AI_OPERATION_NAME] == "update_memory" + assert attrs[GEN_AI_MEMORY_RECORD_ID] == "mem-42" + assert attrs[GEN_AI_PROVIDER_NAME] == "mem0" + + points = _get_duration_metric(metric_reader) + assert len(points) >= 1 + + def test_update_captures_content_when_enabled( + self, tracer_provider, exporter, meter_provider, monkeypatch + ): + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true" + ) + wrapped = mock.Mock(return_value={"status": "ok"}) + wrapper = wrap_memory_update(tracer_provider, meter_provider) + wrapper(wrapped, None, ("mem-42",), {"data": "updated content"}) + + span, attrs = _get_attrs(exporter) + assert attrs[GEN_AI_MEMORY_RECORD_CONTENT] == "updated content" + + +class TestMemoryDelete: + def test_basic_delete( + self, tracer_provider, exporter, meter_provider, metric_reader + ): + wrapped = mock.Mock(return_value=None) + wrapper = wrap_memory_delete(tracer_provider, meter_provider) + wrapper(wrapped, None, ("mem-99",), {}) + + span, attrs = _get_attrs(exporter) + assert span.name == "delete_memory mem0" + assert attrs[GEN_AI_OPERATION_NAME] == "delete_memory" + assert attrs[GEN_AI_MEMORY_RECORD_ID] == "mem-99" + assert attrs[GEN_AI_PROVIDER_NAME] == "mem0" + + points = _get_duration_metric(metric_reader) + assert len(points) >= 1 + + +class TestMemoryDeleteAll: + def test_delete_all_with_user( + self, tracer_provider, exporter, meter_provider + ): + wrapped = mock.Mock(return_value=None) + wrapper = wrap_memory_delete_all(tracer_provider, meter_provider) + wrapper(wrapped, None, (), {"user_id": "alice"}) + + span, attrs = _get_attrs(exporter) + assert span.name == "delete_memory mem0" + assert attrs[GEN_AI_OPERATION_NAME] == "delete_memory" + assert attrs[GEN_AI_PROVIDER_NAME] == "mem0" + + +class TestMemoryGetAll: + def test_get_all( + self, tracer_provider, exporter, meter_provider, metric_reader + ): + wrapped = mock.Mock(return_value={"results": [{"id": "a"}]}) + wrapper = wrap_memory_get_all(tracer_provider, meter_provider) + wrapper(wrapped, None, (), {"user_id": "alice"}) + + span, attrs = _get_attrs(exporter) + assert span.name == "search_memory mem0" + assert attrs[GEN_AI_OPERATION_NAME] == "search_memory" + assert attrs[GEN_AI_MEMORY_SEARCH_RESULT_COUNT] == 1 + assert attrs[GEN_AI_PROVIDER_NAME] == "mem0" + + points = _get_duration_metric(metric_reader) + assert len(points) >= 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/CHANGELOG.md index 95f69d6ded..90f8c11e01 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/CHANGELOG.md @@ -6,8 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +- Align AgentSpanData test stubs and span processor with real OpenAI Agents SDK; + remove non-existent `operation`, `description`, `agent_id`, and `model` fields. + ([#4229](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4229)) - Document official package metadata and README for the OpenAI Agents instrumentation. ([#3859](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3859)) +- Populate instructions and tool definitions from Response obj. + ([#4196](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4196)) ## Version 0.1.0 (2025-10-15) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/examples/zero-code/.env.example b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/examples/zero-code/.env.example index 8f39668502..9e2aeb3023 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/examples/zero-code/.env.example +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/examples/zero-code/.env.example @@ -7,8 +7,5 @@ OPENAI_API_KEY=sk-YOUR_API_KEY OTEL_SERVICE_NAME=opentelemetry-python-openai-agents-zero-code -# Enable auto-instrumentation for logs if desired -OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true - # Optionally override the agent name reported on spans # OTEL_GENAI_AGENT_NAME=Travel Concierge diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/src/opentelemetry/instrumentation/openai_agents/span_processor.py b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/src/opentelemetry/instrumentation/openai_agents/span_processor.py index d1dce8ec5e..74be663701 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/src/opentelemetry/instrumentation/openai_agents/span_processor.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/src/opentelemetry/instrumentation/openai_agents/span_processor.py @@ -1056,10 +1056,20 @@ def _build_content_payload(self, span: Span[Any]) -> ContentPayload: elif _is_instance_of(span_data, ResponseSpanData): span_input = getattr(span_data, "input", None) + response_obj = getattr(span_data, "response", None) if capture_messages and span_input: payload.input_messages = ( self._normalize_messages_to_role_parts(span_input) ) + + if ( + capture_system + and response_obj + and hasattr(response_obj, "instructions") + ): + payload.system_instructions = self._normalize_to_text_parts( + response_obj.instructions + ) if capture_system and span_input: sys_instr = self._collect_system_instructions(span_input) if sys_instr: @@ -1519,17 +1529,8 @@ def _get_operation_name(self, span_data: Any) -> str: return GenAIOperationName.CHAT return GenAIOperationName.TEXT_COMPLETION if _is_instance_of(span_data, AgentSpanData): - # Could be create_agent or invoke_agent based on context - operation = getattr(span_data, "operation", None) - normalized = ( - operation.strip().lower() - if isinstance(operation, str) - else None - ) - if normalized in {"create", "create_agent"}: - return GenAIOperationName.CREATE_AGENT - if normalized in {"invoke", "invoke_agent"}: - return GenAIOperationName.INVOKE_AGENT + # The OpenAI Agents SDK AgentSpanData has no "operation" field; + # agent spans always represent invoke_agent. return GenAIOperationName.INVOKE_AGENT if _is_instance_of(span_data, FunctionSpanData): return GenAIOperationName.EXECUTE_TOOL @@ -1831,24 +1832,20 @@ def _get_attributes_from_agent_span_data( if name: yield GEN_AI_AGENT_NAME, name - agent_id = ( - self.agent_id - or getattr(span_data, "agent_id", None) - or self._agent_id_default - ) + # agent_id and description are not available on the OpenAI Agents SDK + # AgentSpanData; only use user-configured overrides. + agent_id = self.agent_id or self._agent_id_default if agent_id: yield GEN_AI_AGENT_ID, agent_id - description = ( - self.agent_description - or getattr(span_data, "description", None) - or self._agent_description_default - ) + description = self.agent_description or self._agent_description_default if description: yield GEN_AI_AGENT_DESCRIPTION, description - model = getattr(span_data, "model", None) - if not model and agent_content: + # The OpenAI Agents SDK AgentSpanData has no "model" field; fall back to + # the model aggregated from child generation/response spans. + model = None + if agent_content: model = agent_content.get("request_model") if model: yield GEN_AI_REQUEST_MODEL, model @@ -2029,6 +2026,22 @@ def _get_attributes_from_response_span_data( if output_tokens is not None: yield GEN_AI_USAGE_OUTPUT_TOKENS, output_tokens + # Tool definitions from response + if self._capture_tool_definitions and hasattr( + span_data.response, "tools" + ): + yield ( + GEN_AI_TOOL_DEFINITIONS, + safe_json_dumps( + list( + map( + lambda tool: tool.to_dict(), + span_data.response.tools, + ) + ) + ), + ) + # Input/output messages if ( self.include_sensitive_data diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/stubs/agents/tracing/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/stubs/agents/tracing/__init__.py index 4ed06c8977..509fd537b3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/stubs/agents/tracing/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/stubs/agents/tracing/__init__.py @@ -35,12 +35,9 @@ @dataclass class AgentSpanData: name: str | None = None + handoffs: list[str] | None = None tools: list[str] | None = None output_type: str | None = None - description: str | None = None - agent_id: str | None = None - model: str | None = None - operation: str | None = None @property def type(self) -> str: @@ -200,8 +197,16 @@ def generation_span(**kwargs: Any): @contextmanager -def agent_span(**kwargs: Any): - data = AgentSpanData(**kwargs) +def agent_span( + name: str, + handoffs: list[str] | None = None, + tools: list[str] | None = None, + output_type: str | None = None, + **kwargs: Any, +): + data = AgentSpanData( + name=name, handoffs=handoffs, tools=tools, output_type=output_type + ) span = _PROVIDER.create_span(data, parent=_CURRENT_TRACE) span.start() try: diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_tracer.py b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_tracer.py index 1f21ab25c0..5c62fd492e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_tracer.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_tracer.py @@ -25,6 +25,7 @@ set_trace_processors, trace, ) +from openai.types.responses import FunctionTool # noqa: E402 from opentelemetry.instrumentation.openai_agents import ( # noqa: E402 OpenAIAgentsInstrumentor, @@ -62,6 +63,9 @@ GEN_AI_OUTPUT_MESSAGES = getattr( GenAI, "GEN_AI_OUTPUT_MESSAGES", "gen_ai.output.messages" ) +GEN_AI_TOOL_DEFINITIONS = getattr( + GenAI, "GEN_AI_TOOL_DEFINITIONS", "gen_ai.tool.definitions" +) def _instrument_with_provider(**instrument_kwargs): @@ -171,40 +175,31 @@ def test_function_span_records_tool_attributes(): exporter.clear() -def test_agent_create_span_records_attributes(): +def test_agent_invoke_span_records_attributes(): instrumentor, exporter = _instrument_with_provider() try: with trace("workflow"): with agent_span( - operation="create", name="support_bot", - description="Answers support questions", - agent_id="agt_123", - model="gpt-4o-mini", + handoffs=["escalation_bot"], + tools=["search"], + output_type="str", ): pass spans = exporter.get_finished_spans() - create_span = next( + invoke_span = next( span for span in spans if span.attributes[GenAI.GEN_AI_OPERATION_NAME] - == GenAI.GenAiOperationNameValues.CREATE_AGENT.value + == GenAI.GenAiOperationNameValues.INVOKE_AGENT.value ) - assert create_span.kind is SpanKind.CLIENT - assert create_span.name == "create_agent support_bot" - assert create_span.attributes[GEN_AI_PROVIDER_NAME] == "openai" - assert create_span.attributes[GenAI.GEN_AI_AGENT_NAME] == "support_bot" - assert ( - create_span.attributes[GenAI.GEN_AI_AGENT_DESCRIPTION] - == "Answers support questions" - ) - assert create_span.attributes[GenAI.GEN_AI_AGENT_ID] == "agt_123" - assert ( - create_span.attributes[GenAI.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" - ) + assert invoke_span.kind is SpanKind.CLIENT + assert invoke_span.name == "invoke_agent support_bot" + assert invoke_span.attributes[GEN_AI_PROVIDER_NAME] == "openai" + assert invoke_span.attributes[GenAI.GEN_AI_AGENT_NAME] == "support_bot" finally: instrumentor.uninstrument() exporter.clear() @@ -425,7 +420,7 @@ def test_agent_name_override_applied_to_agent_spans(): try: with trace("workflow"): - with agent_span(operation="invoke", name="support_bot"): + with agent_span(name="support_bot"): pass spans = exporter.get_finished_spans() @@ -487,8 +482,26 @@ def __init__(self, input_tokens: int, output_tokens: int) -> None: class _Response: def __init__(self) -> None: self.id = "resp-123" + self.instructions = "You are a helpful assistant." self.model = "gpt-4o-mini" self.usage = _Usage(42, 9) + self.tools = [ + FunctionTool( + name="get_current_weather", + type="function", + description="Get the current weather in a given location", + parameters={ + "type": "object", + "properties": { + "location": { + "title": "Location", + "type": "string", + }, + }, + "required": ["location"], + }, + ) + ] self.output = [{"finish_reason": "stop"}] try: @@ -516,6 +529,30 @@ def __init__(self) -> None: assert response.attributes[GenAI.GEN_AI_RESPONSE_FINISH_REASONS] == ( "stop", ) + + system_instructions = json.loads( + response.attributes[GenAI.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + assert system_instructions == [ + {"type": "text", "content": "You are a helpful assistant."} + ] + tool_definitions = json.loads( + response.attributes[GEN_AI_TOOL_DEFINITIONS] + ) + assert tool_definitions == [ + { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"title": "Location", "type": "string"}, + }, + "required": ["location"], + }, + } + ] finally: instrumentor.uninstrument() exporter.clear() diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_z_span_processor_unit.py b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_z_span_processor_unit.py index b2c8c7c8f3..c879aa06a1 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_z_span_processor_unit.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-agents-v2/tests/test_z_span_processor_unit.py @@ -156,19 +156,14 @@ def test_operation_and_span_naming(processor_setup): == sp.GenAIOperationName.EMBEDDINGS ) - agent_create = AgentSpanData(operation=" CREATE ") + # AgentSpanData always maps to invoke_agent (no operation field in OpenAI Agents SDK) + agent_data = AgentSpanData(name="bot") assert ( - processor._get_operation_name(agent_create) - == sp.GenAIOperationName.CREATE_AGENT - ) - - agent_invoke = AgentSpanData(operation="invoke_agent") - assert ( - processor._get_operation_name(agent_invoke) + processor._get_operation_name(agent_data) == sp.GenAIOperationName.INVOKE_AGENT ) - agent_default = AgentSpanData(operation=None) + agent_default = AgentSpanData() assert ( processor._get_operation_name(agent_default) == sp.GenAIOperationName.INVOKE_AGENT @@ -315,26 +310,19 @@ def __init__(self) -> None: agent_span = AgentSpanData( name="helper", output_type="json", - description="desc", - agent_id="agent-123", - model="model-x", - operation="invoke_agent", ) agent_attrs = _collect( processor._get_attributes_from_agent_span_data(agent_span, None) ) assert agent_attrs[sp.GEN_AI_AGENT_NAME] == "helper" - assert agent_attrs[sp.GEN_AI_AGENT_ID] == "agent-123" - assert agent_attrs[sp.GEN_AI_REQUEST_MODEL] == "model-x" + assert sp.GEN_AI_AGENT_ID not in agent_attrs + assert sp.GEN_AI_REQUEST_MODEL not in agent_attrs assert agent_attrs[sp.GEN_AI_OUTPUT_TYPE] == sp.GenAIOutputType.TEXT # Fallback to aggregated model when span data lacks it agent_span_no_model = AgentSpanData( name="helper-2", output_type="json", - description="desc", - agent_id="agent-456", - operation="invoke_agent", ) agent_content = { "input_messages": [], @@ -435,9 +423,7 @@ def test_span_lifecycle_and_shutdown(processor_setup): parent_span = FakeSpan( trace_id="trace-1", span_id="span-1", - span_data=AgentSpanData( - operation="invoke", name="agent", model="gpt-4o" - ), + span_data=AgentSpanData(name="agent"), started_at="2024-01-01T00:00:00Z", ended_at="2024-01-01T00:00:02Z", ) @@ -476,7 +462,7 @@ def test_span_lifecycle_and_shutdown(processor_setup): linger_span = FakeSpan( trace_id="trace-2", span_id="span-3", - span_data=AgentSpanData(operation=None), + span_data=AgentSpanData(), started_at="2024-01-01T00:00:06Z", ) processor.on_span_start(linger_span) @@ -518,7 +504,6 @@ def test_chat_span_renamed_with_model(processor_setup): trace_id=trace.trace_id, span_id="agent-span", span_data=AgentSpanData( - operation="invoke_agent", name="Agent", ), started_at="2025-01-01T00:00:00Z",